Merge pull request #9549 from yyforyongyu/fix-bitcond-test

Fix unit test flake `TestHistoricalConfDetailsTxIndex`

Makefile

@@ -276,18 +276,18 @@ unit-bench: $(BTCD_BIN)
 # FLAKE HUNTING
 # =============
 
-#? flakehunter: Run the integration tests continuously until one fails
+#? flakehunter-itest: Run the integration tests continuously until one fails
-flakehunter: build-itest
+flakehunter-itest: build-itest
 	@$(call print, "Flake hunting ${backend} integration tests.")
 	while [ $$? -eq 0 ]; do make itest-only icase='${icase}' backend='${backend}'; done
 
-#? flake-unit: Run the unit tests continuously until one fails
+#? flakehunter-unit: Run the unit tests continuously until one fails
-flake-unit:
+flakehunter-unit:
-	@$(call print, "Flake hunting unit tests.")
+	@$(call print, "Flake hunting unit test.")
-	while [ $$? -eq 0 ]; do GOTRACEBACK=all $(UNIT) -count=1; done
+	scripts/unit-test-flake-hunter.sh ${pkg} ${case}
 
-#? flakehunter-parallel: Run the integration tests continuously until one fails, running up to ITEST_PARALLELISM test tranches in parallel (default 4)
+#? flakehunter-itest-parallel: Run the integration tests continuously until one fails, running up to ITEST_PARALLELISM test tranches in parallel (default 4)
-flakehunter-parallel:
+flakehunter-itest-parallel:
 	@$(call print, "Flake hunting ${backend} integration tests in parallel.")
 	while [ $$? -eq 0 ]; do make itest-parallel tranches=1 parallel=${ITEST_PARALLELISM} icase='${icase}' backend='${backend}'; done
 

chainntnfs/bitcoindnotify/bitcoind_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/integration/rpctest"
+	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/btcsuite/btcwallet/chain"
 	"github.com/lightningnetwork/lnd/blockcache"
 	"github.com/lightningnetwork/lnd/chainntnfs"
@@ -103,6 +104,54 @@ func syncNotifierWithMiner(t *testing.T, notifier *BitcoindNotifier,
 				"err=%v, minerHeight=%v, bitcoindHeight=%v",
 				err, minerHeight, bitcoindHeight)
 		}
+
+		// Get the num of connections the miner has. We expect it to
+		// have at least one connection with the chain backend.
+		count, err := miner.Client.GetConnectionCount()
+		require.NoError(t, err)
+		if count != 0 {
+			continue
+		}
+
+		// Reconnect the miner and the chain backend.
+		//
+		// NOTE: The connection should have been made before we perform
+		// the `syncNotifierWithMiner`. However, due to an unknown
+		// reason, the miner may refuse to process the inbound
+		// connection made by the bitcoind node, causing the connection
+		// to fail. It's possible there's a bug in the handshake between
+		// the two nodes.
+		//
+		// A normal flow is, bitcoind starts a v2 handshake flow, which
+		// btcd will fail and disconnect. Upon seeing this
+		// disconnection, bitcoind will try a v1 handshake and succeeds.
+		// The failed flow is, upon seeing the v2 handshake, btcd
+		// doesn't seem to perform the disconnect. Instead an EOF
+		// websocket error is found.
+		//
+		// TODO(yy): Fix the above bug in `btcd`. This can be reproduced
+		// using `make flakehunter-unit pkg=$pkg case=$case`, with,
+		// `case=TestHistoricalConfDetailsNoTxIndex/rpc_polling_enabled`
+		// `pkg=chainntnfs/bitcoindnotify`.
+		// Also need to modify the temp dir logic so we can save the
+		// debug logs.
+		// This bug is likely to be fixed when we implement the
+		// encrypted p2p conn, or when we properly fix the shutdown
+		// issues in all our RPC conns.
+		t.Log("Expected to the chain backend to have one conn with " +
+			"the miner, instead it's disconnected!")
+
+		// We now ask the miner to add the chain backend back.
+		host := fmt.Sprintf(
+			"127.0.0.1:%s", notifier.chainParams.DefaultPort,
+		)
+
+		// NOTE:AddNode must take a host that has the format
+		// `host:port`, otherwise the default port will be used. Check
+		// `normalizeAddress` in btcd for details.
+		err = miner.Client.AddNode(host, rpcclient.ANAdd)
+		require.NoError(t, err, "Failed to connect miner to the chain "+
+			"backend")
 	}
 }
 
@@ -130,7 +179,7 @@ func testHistoricalConfDetailsTxIndex(t *testing.T, rpcPolling bool) {
 	)
 
 	bitcoindConn := unittest.NewBitcoindBackend(
-		t, unittest.NetParams, miner.P2PAddress(), true, rpcPolling,
+		t, unittest.NetParams, miner, true, rpcPolling,
 	)
 
 	hintCache := initHintCache(t)
@@ -140,8 +189,6 @@ func testHistoricalConfDetailsTxIndex(t *testing.T, rpcPolling bool) {
 		t, bitcoindConn, hintCache, hintCache, blockCache,
 	)
 
-	syncNotifierWithMiner(t, notifier, miner)
-
 	// A transaction unknown to the node should not be found within the
 	// txindex even if it is enabled, so we should not proceed with any
 	// fallback methods.
@@ -230,13 +277,15 @@ func testHistoricalConfDetailsNoTxIndex(t *testing.T, rpcpolling bool) {
 	miner := unittest.NewMiner(t, unittest.NetParams, nil, true, 25)
 
 	bitcoindConn := unittest.NewBitcoindBackend(
-		t, unittest.NetParams, miner.P2PAddress(), false, rpcpolling,
+		t, unittest.NetParams, miner, false, rpcpolling,
 	)
 
 	hintCache := initHintCache(t)
 	blockCache := blockcache.NewBlockCache(10000)
 
-	notifier := setUpNotifier(t, bitcoindConn, hintCache, hintCache, blockCache)
+	notifier := setUpNotifier(
+		t, bitcoindConn, hintCache, hintCache, blockCache,
+	)
 
 	// Since the node has its txindex disabled, we fall back to scanning the
 	// chain manually. A transaction unknown to the network should not be
@@ -245,7 +294,11 @@ func testHistoricalConfDetailsNoTxIndex(t *testing.T, rpcpolling bool) {
 	copy(unknownHash[:], bytes.Repeat([]byte{0x10}, 32))
 	unknownConfReq, err := chainntnfs.NewConfRequest(&unknownHash, testScript)
 	require.NoError(t, err, "unable to create conf request")
-	broadcastHeight := syncNotifierWithMiner(t, notifier, miner)
+
+	// Get the current best height.
+	_, broadcastHeight, err := miner.Client.GetBestBlock()
+	require.NoError(t, err, "unable to retrieve miner's current height")
+
 	_, txStatus, err := notifier.historicalConfDetails(
 		unknownConfReq, uint32(broadcastHeight), uint32(broadcastHeight),
 	)

chainntnfs/test/test_interface.go

@@ -1932,7 +1932,7 @@ func TestInterfaces(t *testing.T, targetBackEnd string) {
 		case "bitcoind":
 			var bitcoindConn *chain.BitcoindConn
 			bitcoindConn = unittest.NewBitcoindBackend(
-				t, unittest.NetParams, p2pAddr, true, false,
+				t, unittest.NetParams, miner, true, false,
 			)
 			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
 				return bitcoindnotify.New(
@@ -1944,7 +1944,7 @@ func TestInterfaces(t *testing.T, targetBackEnd string) {
 		case "bitcoind-rpc-polling":
 			var bitcoindConn *chain.BitcoindConn
 			bitcoindConn = unittest.NewBitcoindBackend(
-				t, unittest.NetParams, p2pAddr, true, true,
+				t, unittest.NetParams, miner, true, true,
 			)
 			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
 				return bitcoindnotify.New(

docs/release-notes/release-notes-0.19.0.md

@@ -317,6 +317,9 @@ The underlying functionality between those two options remain the same.
   now documented and
   [fixed](https://github.com/lightningnetwork/lnd/pull/9368).
 
+* [Fixed](https://github.com/lightningnetwork/lnd/pull/9549) a long standing
+  unit test flake found in the `chainntnfs/bitcoindnotify` package.
+
 ## Database
 
 * [Migrate the mission control 

lntest/unittest/backend.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/integration/rpctest"
+	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/btcsuite/btcwallet/chain"
 	"github.com/btcsuite/btcwallet/walletdb"
 	"github.com/lightninglabs/neutrino"
@@ -37,9 +39,15 @@ func NewMiner(t *testing.T, netParams *chaincfg.Params, extraArgs []string,
 
 	t.Helper()
 
-	// Add the trickle interval argument to the extra args.
+	args := []string{
-	trickle := fmt.Sprintf("--trickleinterval=%v", TrickleInterval)
+		"--nobanning",
-	extraArgs = append(extraArgs, trickle)
+		"--debuglevel=debug",
+		fmt.Sprintf("--trickleinterval=%v", TrickleInterval),
+
+		// Don't disconnect if a reply takes too long.
+		"--nostalldetect",
+	}
+	extraArgs = append(extraArgs, args...)
 
 	node, err := rpctest.New(netParams, nil, extraArgs, "")
 	require.NoError(t, err, "unable to create backend node")
@@ -73,9 +81,10 @@ func NewMiner(t *testing.T, netParams *chaincfg.Params, extraArgs []string,
 // backend node should maintain a transaction index. The rpcpolling boolean
 // can be set to determine whether bitcoind's RPC polling interface should be
 // used for block and tx notifications or if its ZMQ interface should be used.
-// A connection to the newly spawned bitcoind node is returned.
+// A connection to the newly spawned bitcoind node is returned once the bitcoind
+// is synced to the miner's best height.
 func NewBitcoindBackend(t *testing.T, netParams *chaincfg.Params,
-	minerAddr string, txindex, rpcpolling bool) *chain.BitcoindConn {
+	miner *rpctest.Harness, txindex, rpcpolling bool) *chain.BitcoindConn {
 
 	t.Helper()
 
@@ -88,29 +97,51 @@ func NewBitcoindBackend(t *testing.T, netParams *chaincfg.Params,
 	zmqBlockHost := fmt.Sprintf("tcp://127.0.0.1:%d", zmqBlockPort)
 	zmqTxHost := fmt.Sprintf("tcp://127.0.0.1:%d", zmqTxPort)
 
+	// TODO(yy): Make this configurable via `chain.BitcoindConfig` and
+	// replace the default P2P port when set.
+	p2pPort := port.NextAvailablePort()
+	netParams.DefaultPort = fmt.Sprintf("%d", p2pPort)
+
 	args := []string{
-		"-connect=" + minerAddr,
 		"-datadir=" + tempBitcoindDir,
 		"-regtest",
 		"-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6fd$507c670e800a95" +
 			"284294edb5773b05544b220110063096c221be9933c82d38e1",
 		fmt.Sprintf("-rpcport=%d", rpcPort),
 		fmt.Sprintf("-bind=127.0.0.1:%d=onion", torBindPort),
+		fmt.Sprintf("-port=%d", p2pPort),
 		"-disablewallet",
 		"-zmqpubrawblock=" + zmqBlockHost,
 		"-zmqpubrawtx=" + zmqTxHost,
+
+		// whitelist localhost to speed up relay.
+		"-whitelist=127.0.0.1",
+
+		// Disable v2 transport as btcd doesn't support it yet.
+		//
+		// TODO(yy): Remove this line once v2 conn is supported in
+		// `btcd`.
+		"-v2transport=0",
 	}
 	if txindex {
 		args = append(args, "-txindex")
 	}
 
 	bitcoind := exec.Command("bitcoind", args...)
-	if err := bitcoind.Start(); err != nil {
+	err := bitcoind.Start()
-		t.Fatalf("unable to start bitcoind: %v", err)
+	require.NoError(t, err, "unable to start bitcoind")
-	}
+
 	t.Cleanup(func() {
-		_ = bitcoind.Process.Kill()
+		// Kill `bitcoind` and assert there's no error.
-		_ = bitcoind.Wait()
+		err = bitcoind.Process.Kill()
+		require.NoError(t, err)
+
+		err = bitcoind.Wait()
+		if strings.Contains(err.Error(), "signal: killed") {
+			return
+		}
+
+		require.NoError(t, err)
 	})
 
 	// Wait for the bitcoind instance to start up.
@@ -142,7 +173,7 @@ func NewBitcoindBackend(t *testing.T, netParams *chaincfg.Params,
 	}
 
 	var conn *chain.BitcoindConn
-	err := wait.NoError(func() error {
+	err = wait.NoError(func() error {
 		var err error
 		conn, err = chain.NewBitcoindConn(cfg)
 		if err != nil {
@@ -157,9 +188,131 @@ func NewBitcoindBackend(t *testing.T, netParams *chaincfg.Params,
 	}
 	t.Cleanup(conn.Stop)
 
+	// Assert that the connection with the miner is made.
+	//
+	// Create a new RPC client.
+	rpcCfg := rpcclient.ConnConfig{
+		Host:                 cfg.Host,
+		User:                 cfg.User,
+		Pass:                 cfg.Pass,
+		DisableConnectOnNew:  true,
+		DisableAutoReconnect: false,
+		DisableTLS:           true,
+		HTTPPostMode:         true,
+	}
+
+	rpcClient, err := rpcclient.New(&rpcCfg, nil)
+	require.NoError(t, err, "failed to create RPC client")
+
+	// Connect to the miner node.
+	err = rpcClient.AddNode(miner.P2PAddress(), rpcclient.ANAdd)
+	require.NoError(t, err, "failed to connect to miner")
+
+	// Get the network info and assert the num of outbound connections is 1.
+	err = wait.NoError(func() error {
+		result, err := rpcClient.GetNetworkInfo()
+		require.NoError(t, err)
+
+		if int(result.Connections) != 1 {
+			return fmt.Errorf("want 1 conn, got %d",
+				result.Connections)
+		}
+
+		if int(result.ConnectionsOut) != 1 {
+			return fmt.Errorf("want 1 outbound conn, got %d",
+				result.Connections)
+		}
+
+		return nil
+	}, wait.DefaultTimeout)
+	require.NoError(t, err, "timeout connecting to the miner")
+
+	// Assert the chain backend is synced to the miner.
+	syncBitcoindWithMiner(t, rpcClient, miner, p2pPort)
+
+	// Tear down the rpc client.
+	rpcClient.Shutdown()
+
 	return conn
 }
 
+// syncBitcoindWithMiner waits until the bitcoind node is synced with the miner.
+func syncBitcoindWithMiner(t *testing.T, notifier *rpcclient.Client,
+	miner *rpctest.Harness, p2pPort int) uint32 {
+
+	_, minerHeight, err := miner.Client.GetBestBlock()
+	require.NoError(t, err, "unable to retrieve miner's current height")
+
+	timeout := time.After(10 * time.Second)
+	for {
+		info, err := notifier.GetBlockChainInfo()
+		require.NoError(t, err)
+
+		bitcoindHeight := info.Blocks
+
+		t.Logf("miner height=%v, bitcoind height=%v", minerHeight,
+			bitcoindHeight)
+
+		if bitcoindHeight == minerHeight {
+			return uint32(bitcoindHeight)
+		}
+
+		select {
+		case <-time.After(100 * time.Millisecond):
+		case <-timeout:
+			t.Fatalf("timed out in syncNotifierWithMiner, got "+
+				"err=%v, minerHeight=%v, bitcoindHeight=%v",
+				err, minerHeight, bitcoindHeight)
+		}
+
+		// Get the num of connections the miner has. We expect it to
+		// have at least one connection with the chain backend.
+		count, err := miner.Client.GetConnectionCount()
+		require.NoError(t, err)
+		if count != 0 {
+			continue
+		}
+
+		// Reconnect the miner and the chain backend.
+		//
+		// NOTE: The connection should have been made before we perform
+		// the `syncNotifierWithMiner`. However, due unknown reason, the
+		// miner may refuse to process the inbound connection made by
+		// the bitcoind node, causing the connection to fail. It's
+		// possible there's a bug in the handshake between the two
+		// nodes.
+		//
+		// A normal flow is, bitcoind starts a v2 handshake flow, which
+		// btcd will fail and disconnect. Upon seeing this
+		// disconnection, bitcoind will try a v1 handshake and succeeds.
+		// The failed flow is, upon seeing the v2 handshake, btcd
+		// doesn't seem to perform the disconnect. Instead an EOF
+		// websocket error is found.
+		//
+		// TODO(yy): Fix the above bug in `btcd`. This can be reproduced
+		// using `make flakehunter-unit pkg=$pkg case=$case`, with,
+		// `case=TestHistoricalConfDetailsNoTxIndex/rpc_polling_enabled`
+		// `pkg=chainntnfs/bitcoindnotify`.
+		// Also need to modify the temp dir logic so we can save the
+		// debug logs.
+		// This bug is likely to be fixed when we implement the
+		// encrypted p2p conn, or when we properly fix the shutdown
+		// issues in all our RPC conns.
+		t.Log("Expected to the chain backend to have one conn with " +
+			"the miner, instead it's disconnected!")
+
+		// We now ask the miner to add the chain backend back.
+		host := fmt.Sprintf("127.0.0.1:%d", p2pPort)
+
+		// NOTE:AddNode must take a host that has the format
+		// `host:port`, otherwise the default port will be used. Check
+		// `normalizeAddress` in btcd for details.
+		err = miner.Client.AddNode(host, rpcclient.ANAdd)
+		require.NoError(t, err, "Failed to connect miner to the chain "+
+			"backend")
+	}
+}
+
 // NewNeutrinoBackend spawns a new neutrino node that connects to a miner at
 // the specified address.
 func NewNeutrinoBackend(t *testing.T, netParams *chaincfg.Params,

lnwallet/test/test_interface.go

@@ -3352,8 +3352,7 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 		case "bitcoind":
 			// Start a bitcoind instance.
 			chainConn := unittest.NewBitcoindBackend(
-				t, unittest.NetParams, miningNode.P2PAddress(),
+				t, unittest.NetParams, miningNode, true, false,
-				true, false,
 			)
 
 			// Create a btcwallet bitcoind client for both Alice and
@@ -3364,8 +3363,7 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 		case "bitcoind-rpc-polling":
 			// Start a bitcoind instance.
 			chainConn := unittest.NewBitcoindBackend(
-				t, unittest.NetParams, miningNode.P2PAddress(),
+				t, unittest.NetParams, miningNode, true, true,
-				true, true,
 			)
 
 			// Create a btcwallet bitcoind client for both Alice and

routing/chainview/interface_test.go

@@ -160,9 +160,40 @@ func assertFilteredBlock(t *testing.T, fb *FilteredBlock, expectedHeight int32,
 
 }
 
-func testFilterBlockNotifications(node *rpctest.Harness,
+// setupMinerAndChainView creates a miner node and initialize a chain view using
-	chainView FilteredChainView, chainViewInit chainViewInitFunc,
+// this miner node. It returns the miner and the chain view.
-	t *testing.T) {
+func setupMinerAndChainView(t *testing.T, chainViewInit chainViewInitFunc) (
+	*rpctest.Harness, FilteredChainView) {
+
+	// Initialize the harness around a btcd node which will serve as our
+	// dedicated miner to generate blocks, cause re-orgs, etc. We'll set up
+	// this node with a chain length of 125, so we have plenty of BTC to
+	// play around with.
+	miner := unittest.NewMiner(
+		t, netParams, []string{"--txindex"}, true, 25,
+	)
+	rpcConfig := miner.RPCConfig()
+
+	_, bestHeight, err := miner.Client.GetBestBlock()
+	require.NoError(t, err)
+
+	chainView, err := chainViewInit(t, rpcConfig, miner, bestHeight)
+	require.NoError(t, err)
+
+	err = chainView.Start()
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		require.NoError(t, chainView.Stop())
+	})
+
+	return miner, chainView
+}
+
+func testFilterBlockNotifications(t *testing.T,
+	chainViewInit chainViewInitFunc) {
+
+	node, chainView := setupMinerAndChainView(t, chainViewInit)
 
 	// To start the test, we'll create to fresh outputs paying to the
 	// private key that we generated above.
@@ -270,9 +301,8 @@ func testFilterBlockNotifications(node *rpctest.Harness,
 	}
 }
 
-func testUpdateFilterBackTrack(node *rpctest.Harness,
+func testUpdateFilterBackTrack(t *testing.T, chainViewInit chainViewInitFunc) {
-	chainView FilteredChainView, chainViewInit chainViewInitFunc,
+	node, chainView := setupMinerAndChainView(t, chainViewInit)
-	t *testing.T) {
 
 	// To start, we'll create a fresh output paying to the height generated
 	// above.
@@ -345,8 +375,8 @@ func testUpdateFilterBackTrack(node *rpctest.Harness,
 	}
 }
 
-func testFilterSingleBlock(node *rpctest.Harness, chainView FilteredChainView,
+func testFilterSingleBlock(t *testing.T, chainViewInit chainViewInitFunc) {
-	chainViewInit chainViewInitFunc, t *testing.T) {
+	node, chainView := setupMinerAndChainView(t, chainViewInit)
 
 	// In this test, we'll test the manual filtration of blocks, which can
 	// be used by clients to manually rescan their sub-set of the UTXO set.
@@ -445,9 +475,10 @@ func testFilterSingleBlock(node *rpctest.Harness, chainView FilteredChainView,
 // testFilterBlockDisconnected triggers a reorg all the way back to genesis,
 // and a small 5 block reorg, ensuring the chainView notifies about
 // disconnected and connected blocks in the order we expect.
-func testFilterBlockDisconnected(node *rpctest.Harness,
+func testFilterBlockDisconnected(t *testing.T,
-	chainView FilteredChainView, chainViewInit chainViewInitFunc,
+	chainViewInit chainViewInitFunc) {
-	t *testing.T) {
+
+	node, _ := setupMinerAndChainView(t, chainViewInit)
 
 	// Create a node that has a shorter chain than the main chain, so we
 	// can trigger a reorg.
@@ -460,7 +491,7 @@ func testFilterBlockDisconnected(node *rpctest.Harness,
 
 	// Init a chain view that has this node as its block source.
 	reorgView, err := chainViewInit(
-		t, reorgNode.RPCConfig(), reorgNode.P2PAddress(), bestHeight,
+		t, reorgNode.RPCConfig(), reorgNode, bestHeight,
 	)
 	require.NoError(t, err, "unable to create chain view")
 
@@ -632,12 +663,11 @@ func testFilterBlockDisconnected(node *rpctest.Harness,
 }
 
 type chainViewInitFunc func(t *testing.T, rpcInfo rpcclient.ConnConfig,
-	p2pAddr string, bestHeight int32) (FilteredChainView, error)
+	miner *rpctest.Harness, bestHeight int32) (FilteredChainView, error)
 
 type testCase struct {
 	name string
-	test func(*rpctest.Harness, FilteredChainView, chainViewInitFunc,
+	test func(*testing.T, chainViewInitFunc)
-		*testing.T)
 }
 
 var chainViewTests = []testCase{
@@ -666,12 +696,12 @@ var interfaceImpls = []struct {
 	{
 		name: "bitcoind_zmq",
 		chainViewInit: func(t *testing.T, _ rpcclient.ConnConfig,
-			p2pAddr string, bestHeight int32) (FilteredChainView,
+			miner *rpctest.Harness, bestHeight int32) (
-			error) {
+			FilteredChainView, error) {
 
 			// Start a bitcoind instance.
 			chainConn := unittest.NewBitcoindBackend(
-				t, unittest.NetParams, p2pAddr, true,
+				t, unittest.NetParams, miner, true,
 				false,
 			)
 			blockCache := blockcache.NewBlockCache(10000)
@@ -686,12 +716,12 @@ var interfaceImpls = []struct {
 	{
 		name: "bitcoind_polling",
 		chainViewInit: func(t *testing.T, _ rpcclient.ConnConfig,
-			p2pAddr string, bestHeight int32) (FilteredChainView,
+			miner *rpctest.Harness, bestHeight int32) (
-			error) {
+			FilteredChainView, error) {
 
 			// Wait for the bitcoind instance to start up.
 			chainConn := unittest.NewBitcoindBackend(
-				t, unittest.NetParams, p2pAddr, true,
+				t, unittest.NetParams, miner, true,
 				true,
 			)
 			blockCache := blockcache.NewBlockCache(10000)
@@ -700,14 +730,28 @@ var interfaceImpls = []struct {
 				chainConn, blockCache,
 			)
 
+			// When running in rpc polling mode, the `reorg` method
+			// in `BitcoindClient`'s `ntfnHandler` may be invoked to
+			// handle the last block received from the miner during
+			// bitcoind's startup. This behavior will cause a block
+			// disconnected and a block connected notifications to
+			// be sent to the channels.
+			//
+			// TODO(yy): unify the chain backend logic and put
+			// everything in `btcwallet/chain` instead. The only
+			// place we use this chain view is in `graph/builder`,
+			// in which we subscribe to `FilteredBlocks` and
+			// `DisconnectedBlocks`.
+			time.Sleep(1 * time.Second)
+
 			return chainView, nil
 		},
 	},
 	{
 		name: "p2p_neutrino",
 		chainViewInit: func(t *testing.T, _ rpcclient.ConnConfig,
-			p2pAddr string, bestHeight int32) (FilteredChainView,
+			miner *rpctest.Harness, bestHeight int32) (
-			error) {
+			FilteredChainView, error) {
 
 			spvDir := t.TempDir()
 
@@ -723,7 +767,7 @@ var interfaceImpls = []struct {
 				DataDir:      spvDir,
 				Database:     spvDatabase,
 				ChainParams:  *netParams,
-				ConnectPeers: []string{p2pAddr},
+				ConnectPeers: []string{miner.P2PAddress()},
 			}
 
 			spvNode, err := neutrino.NewChainService(spvConfig)
@@ -776,8 +820,8 @@ var interfaceImpls = []struct {
 	{
 		name: "btcd_websockets",
 		chainViewInit: func(_ *testing.T, config rpcclient.ConnConfig,
-			p2pAddr string, bestHeight int32) (FilteredChainView,
+			_ *rpctest.Harness, _ int32) (
-			error) {
+			FilteredChainView, error) {
 
 			blockCache := blockcache.NewBlockCache(10000)
 			chainView, err := NewBtcdFilteredChainView(
@@ -793,42 +837,17 @@ var interfaceImpls = []struct {
 }
 
 func TestFilteredChainView(t *testing.T) {
-	// Initialize the harness around a btcd node which will serve as our
-	// dedicated miner to generate blocks, cause re-orgs, etc. We'll set up
-	// this node with a chain length of 125, so we have plenty of BTC to
-	// play around with.
-	miner := unittest.NewMiner(
-		t, netParams, []string{"--txindex"}, true, 25,
-	)
-
-	rpcConfig := miner.RPCConfig()
-	p2pAddr := miner.P2PAddress()
-
 	for _, chainViewImpl := range interfaceImpls {
 		t.Logf("Testing '%v' implementation of FilteredChainView",
 			chainViewImpl.name)
 
-		_, bestHeight, err := miner.Client.GetBestBlock()
-		if err != nil {
-			t.Fatalf("error getting best block: %v", err)
-		}
-
-		chainView, err := chainViewImpl.chainViewInit(
-			t, rpcConfig, p2pAddr, bestHeight,
-		)
-		if err != nil {
-			t.Fatalf("unable to make chain view: %v", err)
-		}
-		if err := chainView.Start(); err != nil {
-			t.Fatalf("unable to start chain view: %v", err)
-		}
 		for _, chainViewTest := range chainViewTests {
 			testName := fmt.Sprintf("%v: %v", chainViewImpl.name,
 				chainViewTest.name)
+
 			success := t.Run(testName, func(t *testing.T) {
 				chainViewTest.test(
-					miner, chainView,
+					t, chainViewImpl.chainViewInit,
-					chainViewImpl.chainViewInit, t,
 				)
 			})
 
@@ -837,8 +856,5 @@ func TestFilteredChainView(t *testing.T) {
 			}
 		}
 
-		if err := chainView.Stop(); err != nil {
-			t.Fatalf("unable to stop chain view: %v", err)
-		}
 	}
 }

scripts/unit-test-flake-hunter.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Check if pkg and case variables are provided.
+if [ $# -lt 2 ] || [ $# -gt 3 ]; then
+	echo "Usage: $0 <pkg> <case> [timeout]"
+	exit 1
+fi
+
+pkg=$1
+case=$2
+timeout=${3:-30s} # Default to 30s if not provided.
+
+counter=0
+
+# Run the command in a loop until it fails.
+while output=$(go clean -testcache && make unit-debug log="stdlog trace" pkg=$pkg case=$case timeout=$timeout 2>&1); do
+	((counter++))
+	echo "Test $case passed, count: $counter"
+done
+
+# Only log the output when it fails.
+echo "Test $case failed. Output:"
+echo "$output"