lntest: add bitcoind miner backend

.github/workflows/main.yml

@@ -280,6 +280,8 @@ jobs:
             args: backend=btcd cover=1
           - name: bitcoind
             args: backend=bitcoind cover=1
+          - name: bitcoind-miner
+            args: backend=bitcoind minerbackend=bitcoind
           - name: bitcoind-notxindex
             args: backend="bitcoind notxindex"
           - name: neutrino

.gitignore

@@ -40,6 +40,9 @@ itest/btcd-itest
 itest/.logs-*
 itest/cover
 
+# Local lntest miner logs (dev artifacts)
+lntest/miner/*.log
+
 cmd/cmd
 *.key
 *.hex

itest/lnd_misc_test.go

@@ -1580,7 +1580,7 @@ func testReorgNotifications(ht *lntest.HarnessTest) {
 
 	// Reorg block1.
 	blockHash1 := block1.Header.BlockHash()
-	require.NoError(ht, ht.Miner().Client.InvalidateBlock(&blockHash1))
+	require.NoError(ht, ht.Miner().InvalidateBlock(&blockHash1))
 
 	// Mine empty blocks to evict block1 in bitcoin backend (e.g. bitcoind).
 	ht.Miner().MineEmptyBlocks(2)

itest/lnd_nonstd_sweep_test.go

@@ -123,12 +123,13 @@ func testNonStdSweepInner(ht *lntest.HarnessTest, address string) {
 
 	fee = inputVal - outputVal
 
-	// Fetch the vsize of the transaction so we can determine if the
+	// Calculate the vsize of the transaction so we can determine if the
 	// transaction pays >= 1 sat/vbyte.
-	rawTx := ht.Miner().GetRawTransactionVerbose(txid)
+	weight := ht.CalculateTxWeight(msgTx)
+	vbytes := (int64(weight) + 3) / 4
 
 	// Require fee >= vbytes.
-	require.True(ht, fee >= int(rawTx.Vsize))
+	require.True(ht, int64(fee) >= vbytes)
 
 	// Mine a block to keep the mempool clean.
 	ht.MineBlocksAndAssertNumTxes(1, 1)

itest/lnd_open_channel_test.go

@@ -90,8 +90,7 @@ func testOpenChannelAfterReorg(ht *lntest.HarnessTest) {
 	// open.
 	block := ht.MineBlocksAndAssertNumTxes(10, 1)[0]
 	ht.AssertTxInBlock(block, *fundingTxID)
-	_, err = tempMiner.Client.Generate(15)
-	require.NoError(ht, err, "unable to generate blocks")
+	tempMiner.GenerateBlocks(15)
 
 	// Ensure the chain lengths are what we expect, with the temp miner
 	// being 5 blocks ahead.
@@ -135,8 +134,7 @@ func testOpenChannelAfterReorg(ht *lntest.HarnessTest) {
 
 	// This should have caused a reorg, and Alice should sync to the longer
 	// chain, where the funding transaction is not confirmed.
-	_, tempMinerHeight, err := tempMiner.Client.GetBestBlock()
-	require.NoError(ht, err, "unable to get current blockheight")
+	_, tempMinerHeight := tempMiner.GetBestBlock()
 	ht.WaitForNodeBlockHeight(alice, tempMinerHeight)
 
 	// Since the fundingtx was reorged out, Alice should now have no edges
@@ -1046,8 +1044,7 @@ func testPendingChannelAfterReorg(ht *lntest.HarnessTest) {
 
 	// We now cause a fork, by letting our original miner mine 1 blocks,
 	// and our new miner mine 3.
-	_, err := tempMiner.Client.Generate(3)
-	require.NoError(ht, err, "unable to generate blocks on temp miner")
+	tempMiner.GenerateBlocks(3)
 
 	// Ensure the chain lengths are what we expect, with the temp miner
 	// being 2 blocks ahead.
@@ -1072,8 +1069,7 @@ func testPendingChannelAfterReorg(ht *lntest.HarnessTest) {
 
 	// This should have caused a reorg, and Alice should sync to the longer
 	// chain, where the funding transaction is not confirmed.
-	_, tempMinerHeight, err := tempMiner.Client.GetBestBlock()
-	require.NoError(ht, err, "unable to get current blockheight")
+	_, tempMinerHeight := tempMiner.GetBestBlock()
 	ht.WaitForNodeBlockHeight(alice, tempMinerHeight)
 
 	// After the reorg, the funding transaction's confirmation is removed,

itest/lnd_revocation_test.go

@@ -589,7 +589,7 @@ func revokedCloseRetributionRemoteHodlCase(ht *lntest.HarnessTest,
 			// NOTE: We don't use `ht.GetRawTransaction`
 			// which asserts a txid must be found as the HTLC
 			// spending txes might be aggregated.
-			tx, err := ht.Miner().Client.GetRawTransaction(&txid)
+			tx, err := ht.Miner().GetRawTransactionNoAssert(txid)
 			if err != nil {
 				return nil, err
 			}

itest/lnd_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/btcsuite/btcd/integration/rpctest"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lntest"
+	"github.com/lightningnetwork/lnd/lntest/miner"
 	"github.com/lightningnetwork/lnd/lntest/node"
 	"github.com/lightningnetwork/lnd/lntest/port"
 	"github.com/lightningnetwork/lnd/lntest/wait"
@@ -86,6 +87,12 @@ var (
 	lndExecutable = flag.String(
 		"lndexec", itestLndBinary, "full path to lnd binary",
 	)
+
+	// minerBackendFlag selects which miner backend to use. If not set, a
+	// default miner is used.
+	minerBackendFlag = flag.String(
+		"minerbackend", "", "miner backend (btcd, bitcoind)",
+	)
 )
 
 // TestLightningNetworkDaemon performs a series of integration tests amongst a
@@ -104,8 +111,15 @@ func TestLightningNetworkDaemon(t *testing.T) {
 
 	// Get the binary path and setup the harness test.
 	binary := getLndBinary(t)
-	harnessTest := lntest.SetupHarness(
-		t, binary, *dbBackendFlag, *nativeSQLFlag, feeService,
+	var minerCfg *miner.MinerConfig
+	if minerBackendFlag != nil && *minerBackendFlag != "" {
+		minerCfg = &miner.MinerConfig{
+			Backend: *minerBackendFlag,
+		}
+	}
+
+	harnessTest := lntest.SetupHarnessWithMinerConfig(
+		t, binary, *dbBackendFlag, *nativeSQLFlag, feeService, minerCfg,
 	)
 	defer harnessTest.Stop()
 

lntest/bitcoind_common.go

@@ -4,6 +4,7 @@
 package lntest
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -72,7 +73,20 @@ func (b BitcoindBackendConfig) ConnectMiner() error {
 
 // DisconnectMiner is called to disconnect the miner.
 func (b BitcoindBackendConfig) DisconnectMiner() error {
-	return b.rpcClient.AddNode(b.minerAddr, rpcclient.ANRemove)
+	// `addnode remove` removes from the addnode list, but doesn't reliably
+	// disconnect an existing connection. Use `disconnectnode` first.
+	_, err := b.rpcClient.RawRequest(
+		"disconnectnode",
+		[]json.RawMessage{
+			[]byte(fmt.Sprintf("%q", b.minerAddr)),
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	_ = b.rpcClient.AddNode(b.minerAddr, rpcclient.ANRemove)
+	return nil
 }
 
 // Credentials returns the rpc username, password and host for the backend.
@@ -124,7 +138,8 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string,
 
 	cmdArgs := []string{
 		"-datadir=" + tempBitcoindDir,
-		"-whitelist=127.0.0.1", // whitelist localhost to speed up relay
+		// Whitelist localhost to speed up relay.
+		"-whitelist=127.0.0.1",
 		"-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f" +
 			"d$507c670e800a95284294edb5773b05544b" +
 			"220110063096c221be9933c82d38e1",

lntest/harness.go

@@ -1,6 +1,7 @@
 package lntest
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"runtime/debug"
@@ -54,8 +55,15 @@ const (
 	// mining blocks.
 	maxBlocksAllowed = 100
 
-	finalCltvDelta  = routing.MinCLTVDelta // 18.
-	thawHeightDelta = finalCltvDelta * 2   // 36.
+	// finalCltvDelta is the min CLTV delta used by the router.
+	//
+	// At the time of writing, this is 18 blocks (routing.MinCLTVDelta).
+	finalCltvDelta = routing.MinCLTVDelta
+
+	// thawHeightDelta defines how far in the future we pick thaw heights.
+	//
+	// At the time of writing, this is 36 blocks (finalCltvDelta * 2).
+	thawHeightDelta = finalCltvDelta * 2
 )
 
 var (
@@ -98,7 +106,8 @@ type HarnessTest struct {
 	// runCtx is a context with cancel method. It's used to signal when the
 	// node needs to quit, and used as the parent context when spawning
 	// children contexts for RPC requests.
-	runCtx context.Context //nolint:containedctx
+	//nolint:containedctx
+	runCtx context.Context
 	cancel context.CancelFunc
 
 	// stopChainBackend points to the cleanup function returned by the
@@ -506,8 +515,7 @@ func (h *HarnessTest) NewNode(name string,
 	require.NoError(h, err, "failed to start node %s", node.Name())
 
 	// Get the miner's best block hash.
-	bestBlock, err := h.miner.Client.GetBestBlockHash()
-	require.NoError(h, err, "unable to get best block hash")
+	bestBlock, _ := h.miner.GetBestBlock()
 
 	// Wait until the node's chain backend is synced to the miner's best
 	// block.
@@ -1333,7 +1341,8 @@ func (h *HarnessTest) CloseChannelAssertPending(hn *node.HarnessNode,
 			return nil, nil
 		}
 
-		pendingClose, ok := event.Update.(*lnrpc.CloseStatusUpdate_ClosePending) //nolint:ll
+		//nolint:ll
+		pendingClose, ok := event.Update.(*lnrpc.CloseStatusUpdate_ClosePending)
 		require.Truef(h, ok, "expected channel close "+
 			"update, instead got %v", pendingClose)
 
@@ -2265,10 +2274,14 @@ func (h *HarnessTest) GetOutputIndex(txid chainhash.Hash, addr string) int {
 	p2trOutputIndex := -1
 	for i, txOut := range tx.MsgTx().TxOut {
 		_, addrs, _, err := txscript.ExtractPkScriptAddrs(
-			txOut.PkScript, h.miner.ActiveNet,
+			txOut.PkScript, miner.HarnessNetParams,
 		)
 		require.NoError(h, err)
 
+		if len(addrs) == 0 {
+			continue
+		}
+
 		if addrs[0].String() == addr {
 			p2trOutputIndex = i
 		}
@@ -2575,14 +2588,43 @@ func (h *HarnessTest) DeriveFundingShim(alice, bob *node.HarnessNode,
 	}
 
 	var txid *chainhash.Hash
+	var outputIndex uint32
 	targetOutputs := []*wire.TxOut{fundingOutput}
+
+	findFundingOutputIndex := func(tx *wire.MsgTx) uint32 {
+		for i, out := range tx.TxOut {
+			if out.Value != fundingOutput.Value {
+				continue
+			}
+			if !bytes.Equal(out.PkScript, fundingOutput.PkScript) {
+				continue
+			}
+
+			return uint32(i)
+		}
+
+		require.Failf(
+			h, "funding output not found",
+			"funding output not found in tx %v", txid,
+		)
+
+		return 0
+	}
+
 	if publish {
 		txid = h.SendOutputsWithoutChange(targetOutputs, 5)
+
+		// If we published the funding transaction, then we need to
+		// look it up in the mempool to locate the actual output
+		// index.
+		tx := h.GetRawTransaction(*txid).MsgTx()
+		outputIndex = findFundingOutputIndex(tx)
 	} else {
 		tx := h.CreateTransaction(targetOutputs, 5)
 
 		txHash := tx.TxHash()
 		txid = &txHash
+		outputIndex = findFundingOutputIndex(tx)
 	}
 
 	// At this point, we can being our external channel funding workflow.
@@ -2597,6 +2639,7 @@ func (h *HarnessTest) DeriveFundingShim(alice, bob *node.HarnessNode,
 		FundingTxid: &lnrpc.ChannelPoint_FundingTxidBytes{
 			FundingTxidBytes: txid[:],
 		},
+		OutputIndex: outputIndex,
 	}
 	chanPointShim := &lnrpc.ChanPointShim{
 		Amt:       int64(chanSize),

lntest/harness_miner.go

@@ -321,7 +321,9 @@ func (h *HarnessTest) AssertMinerBlockHeightDelta(
 func (h *HarnessTest) SendRawTransaction(tx *wire.MsgTx,
 	allowHighFees bool) (chainhash.Hash, error) {
 
-	txid, err := h.miner.Client.SendRawTransaction(tx, allowHighFees)
+	// Use the miner's SendRawTransaction method which handles both
+	// btcd and bitcoind backends.
+	txid, err := h.miner.SendRawTransaction(tx, allowHighFees)
 	require.NoError(h, err)
 
 	return *txid, nil

lntest/harness_setup.go

@@ -22,6 +22,18 @@ import (
 func SetupHarness(t *testing.T, binaryPath, dbBackendName string,
 	nativeSQL bool, feeService WebFeeService) *HarnessTest {
 
+	return SetupHarnessWithMinerConfig(
+		t, binaryPath, dbBackendName, nativeSQL, feeService, nil,
+	)
+}
+
+// SetupHarnessWithMinerConfig is identical to SetupHarness, but allows callers
+// to supply a miner configuration. This can be used to select alternative miner
+// backends (e.g. bitcoind) without relying on environment variables.
+func SetupHarnessWithMinerConfig(t *testing.T, binaryPath,
+	dbBackendName string, nativeSQL bool, feeService WebFeeService,
+	minerCfg *miner.MinerConfig) *HarnessTest {
+
 	t.Log("Setting up HarnessTest...")
 
 	// Parse testing flags that influence our test execution.
@@ -36,7 +48,7 @@ func SetupHarness(t *testing.T, binaryPath, dbBackendName string,
 
 	// Init the miner.
 	t.Log("Prepare the miner and mine blocks to activate segwit...")
-	miner := prepareMiner(ht.runCtx, ht.T)
+	miner := prepareMiner(ht.runCtx, ht.T, minerCfg)
 
 	// Start a chain backend.
 	chainBackend, cleanUp := prepareChainBackend(t, miner.P2PAddress())
@@ -59,27 +71,40 @@ func SetupHarness(t *testing.T, binaryPath, dbBackendName string,
 	return ht
 }
 
-// prepareMiner creates an instance of the btcd's rpctest.Harness that will act
-// as the miner for all tests. This will be used to fund the wallets of the
-// nodes within the test network and to drive blockchain related events within
-// the network. Revert the default setting of accepting non-standard
-// transactions on simnet to reject them. Transactions on the lightning network
-// should always be standard to get better guarantees of getting included in to
-// blocks.
-func prepareMiner(ctxt context.Context, t *testing.T) *miner.HarnessMiner {
-	m := miner.NewMiner(ctxt, t)
-
-	// Before we start anything, we want to overwrite some of the
-	// connection settings to make the tests more robust. We might need to
-	// restart the miner while there are already blocks present, which will
-	// take a bit longer than the 1 second the default settings amount to.
-	// Doubling both values will give us retries up to 4 seconds.
-	m.MaxConnRetries = rpctest.DefaultMaxConnectionRetries * 2
-	m.ConnectionRetryTimeout = rpctest.DefaultConnectionRetryTimeout * 2
-
-	// Set up miner and connect chain backend to it.
-	require.NoError(t, m.SetUp(true, 50))
-	require.NoError(t, m.Client.NotifyNewTransactions(false))
+// prepareMiner creates an instance of the miner that will act as the miner
+// for all tests. This will be used to fund the wallets of the nodes within
+// the test network and to drive blockchain related events within the network.
+func prepareMiner(ctxt context.Context, t *testing.T,
+	minerCfg *miner.MinerConfig) *miner.HarnessMiner {
+
+	var m *miner.HarnessMiner
+	switch {
+	case minerCfg != nil:
+		t.Logf("Using miner backend=%s", minerCfg.Backend)
+		m = miner.NewMinerWithConfig(ctxt, t, minerCfg)
+
+	default:
+		// Default to btcd for backward compatibility.
+		t.Log("Using miner backend=btcd")
+		m = miner.NewMiner(ctxt, t)
+	}
+
+	// For btcd, we can optimize connection settings.
+	//
+	// Before we start anything, we want to overwrite some of the connection
+	// settings to make the tests more robust. We might need to restart the
+	// miner while there are already blocks present, which will take a bit
+	// longer than the 1 second the default settings amount to. Doubling
+	// both values will give us retries up to 4 seconds.
+	if m.Harness != nil {
+		m.MaxConnRetries = rpctest.DefaultMaxConnectionRetries * 2
+		m.ConnectionRetryTimeout =
+			rpctest.DefaultConnectionRetryTimeout * 2
+	}
+
+	// Start the miner.
+	require.NoError(t, m.Start(true, 50))
+	require.NoError(t, m.NotifyNewTransactions(false))
 
 	// Next mine enough blocks in order for segwit and the CSV package
 	// soft-fork to activate on SimNet.