Merge pull request #6052 from yyforyongyu/itest-2-refactor-1

itest: prepare itest for major refactor

lntest/harness_miner.go

@@ -0,0 +1,161 @@
+package lntest
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/integration/rpctest"
+	"github.com/btcsuite/btcd/rpcclient"
+)
+
+const (
+	// minerLogFilename is the default log filename for the miner node.
+	minerLogFilename = "output_btcd_miner.log"
+
+	// minerLogDir is the default log dir for the miner node.
+	minerLogDir = ".minerlogs"
+)
+
+var harnessNetParams = &chaincfg.RegressionNetParams
+
+type HarnessMiner struct {
+	*rpctest.Harness
+
+	// runCtx is a context with cancel method. It's used to signal when the
+	// node needs to quit, and used as the parent context when spawning
+	runCtx context.Context
+	cancel context.CancelFunc
+
+	// logPath is the directory path of the miner's logs.
+	logPath string
+
+	// logFilename is the saved log filename of the miner node.
+	logFilename string
+}
+
+// NewMiner creates a new miner using btcd backend with the default log file
+// dir and name.
+func NewMiner() (*HarnessMiner, error) {
+	return newMiner(minerLogDir, minerLogFilename)
+}
+
+// NewTempMiner creates a new miner using btcd backend with the specified log
+// file dir and name.
+func NewTempMiner(tempDir, tempLogFilename string) (*HarnessMiner, error) {
+	return newMiner(tempDir, tempLogFilename)
+}
+
+// newMiner creates a new miner using btcd's rpctest.
+func newMiner(minerDirName, logFilename string) (*HarnessMiner, error) {
+	handler := &rpcclient.NotificationHandlers{}
+	btcdBinary := GetBtcdBinary()
+	baseLogPath := fmt.Sprintf("%s/%s", GetLogDir(), minerDirName)
+
+	args := []string{
+		"--rejectnonstd",
+		"--txindex",
+		"--nowinservice",
+		"--nobanning",
+		"--debuglevel=debug",
+		"--logdir=" + baseLogPath,
+		"--trickleinterval=100ms",
+		// Don't disconnect if a reply takes too long.
+		"--nostalldetect",
+	}
+
+	miner, err := rpctest.New(harnessNetParams, handler, args, btcdBinary)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create mining node: %v", err)
+	}
+
+	ctxt, cancel := context.WithCancel(context.Background())
+	m := &HarnessMiner{
+		Harness:     miner,
+		runCtx:      ctxt,
+		cancel:      cancel,
+		logPath:     baseLogPath,
+		logFilename: logFilename,
+	}
+	return m, nil
+}
+
+// Stop shuts down the miner and saves its logs.
+func (h *HarnessMiner) Stop() error {
+	h.cancel()
+
+	if err := h.TearDown(); err != nil {
+		return fmt.Errorf("tear down miner got error: %s", err)
+	}
+
+	return h.saveLogs()
+}
+
+// saveLogs copies the node logs and save it to the file specified by
+// h.logFilename.
+func (h *HarnessMiner) saveLogs() error {
+	// After shutting down the miner, we'll make a copy of the log files
+	// before deleting the temporary log dir.
+	path := fmt.Sprintf("%s/%s", h.logPath, harnessNetParams.Name)
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		return fmt.Errorf("unable to read log directory: %v", err)
+	}
+
+	for _, file := range files {
+		newFilename := strings.Replace(
+			file.Name(), "btcd.log", h.logFilename, 1,
+		)
+		copyPath := fmt.Sprintf("%s/../%s", h.logPath, newFilename)
+
+		logFile := fmt.Sprintf("%s/%s", path, file.Name())
+		err := CopyFile(filepath.Clean(copyPath), logFile)
+		if err != nil {
+			return fmt.Errorf("unable to copy file: %v", err)
+		}
+	}
+
+	if err = os.RemoveAll(h.logPath); err != nil {
+		return fmt.Errorf("cannot remove dir %s: %v", h.logPath, err)
+	}
+
+	return nil
+}
+
+// waitForTxInMempool blocks until the target txid is seen in the mempool. If
+// the transaction isn't seen within the network before the passed timeout,
+// then an error is returned.
+func (h *HarnessMiner) waitForTxInMempool(txid chainhash.Hash) error {
+	ticker := time.NewTicker(50 * time.Millisecond)
+	defer ticker.Stop()
+
+	var mempool []*chainhash.Hash
+	for {
+		select {
+		case <-h.runCtx.Done():
+			return fmt.Errorf("NetworkHarness has been torn down")
+		case <-time.After(DefaultTimeout):
+			return fmt.Errorf("wanted %v, found %v txs "+
+				"in mempool: %v", txid, len(mempool), mempool)
+
+		case <-ticker.C:
+			var err error
+			mempool, err = h.Client.GetRawMempool()
+			if err != nil {
+				return err
+			}
+
+			for _, mempoolTx := range mempool {
+				if *mempoolTx == txid {
+					return nil
+				}
+			}
+		}
+	}
+}

lntest/harness_net.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
-	"github.com/btcsuite/btcd/integration/rpctest"
 	"github.com/btcsuite/btcd/txscript"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
@@ -55,7 +54,7 @@ type NetworkHarness struct {
 
 	// Miner is a reference to a running full node that can be used to
 	// create new blocks on the network.
-	Miner *rpctest.Harness
+	Miner *HarnessMiner
 
 	// BackendCfg houses the information necessary to use a node as LND
 	// chain backend, such as rpc configuration, P2P information etc.
@@ -94,7 +93,7 @@ type NetworkHarness struct {
 // TODO(roasbeef): add option to use golang's build library to a binary of the
 // current repo. This will save developers from having to manually `go install`
 // within the repo each time before changes
-func NewNetworkHarness(r *rpctest.Harness, b BackendConfig, lndBinary string,
+func NewNetworkHarness(m *HarnessMiner, b BackendConfig, lndBinary string,
 	dbBackend DatabaseBackend) (*NetworkHarness, error) {
 
 	feeService := startFeeService()
@@ -105,8 +104,8 @@ func NewNetworkHarness(r *rpctest.Harness, b BackendConfig, lndBinary string,
 		activeNodes:  make(map[int]*HarnessNode),
 		nodesByPub:   make(map[string]*HarnessNode),
 		lndErrorChan: make(chan error),
-		netParams:    r.ActiveNet,
-		Miner:        r,
+		netParams:    m.ActiveNet,
+		Miner:        m,
 		BackendCfg:   b,
 		feeService:   feeService,
 		runCtx:       ctxt,
@@ -949,41 +948,6 @@ func saveProfilesPage(node *HarnessNode) error {
 	return nil
 }
 
-// waitForTxInMempool blocks until the target txid is seen in the mempool. If
-// the transaction isn't seen within the network before the passed timeout,
-// then an error is returned.
-func (n *NetworkHarness) waitForTxInMempool(txid chainhash.Hash) error {
-	ticker := time.NewTicker(50 * time.Millisecond)
-	defer ticker.Stop()
-
-	ctxt, cancel := context.WithTimeout(n.runCtx, DefaultTimeout)
-	defer cancel()
-
-	var mempool []*chainhash.Hash
-	for {
-		select {
-		case <-n.runCtx.Done():
-			return fmt.Errorf("NetworkHarness has been torn down")
-		case <-ctxt.Done():
-			return fmt.Errorf("wanted %v, found %v txs "+
-				"in mempool: %v", txid, len(mempool), mempool)
-
-		case <-ticker.C:
-			var err error
-			mempool, err = n.Miner.Client.GetRawMempool()
-			if err != nil {
-				return err
-			}
-
-			for _, mempoolTx := range mempool {
-				if *mempoolTx == txid {
-					return nil
-				}
-			}
-		}
-	}
-}
-
 // OpenChannelParams houses the params to specify when opening a new channel.
 type OpenChannelParams struct {
 	// Amt is the local amount being put into the channel.
@@ -1332,7 +1296,7 @@ func (n *NetworkHarness) CloseChannel(lnNode *HarnessNode,
 			return fmt.Errorf("unable to decode closeTxid: "+
 				"%v", err)
 		}
-		if err := n.waitForTxInMempool(*closeTxid); err != nil {
+		if err := n.Miner.waitForTxInMempool(*closeTxid); err != nil {
 			return fmt.Errorf("error while waiting for "+
 				"broadcast tx: %v", err)
 		}
@@ -1601,36 +1565,6 @@ func (n *NetworkHarness) SetFeeEstimateWithConf(
 	n.feeService.setFeeWithConf(fee, conf)
 }
 
-// CopyFile copies the file src to dest.
-func CopyFile(dest, src string) error {
-	s, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer s.Close()
-
-	d, err := os.Create(dest)
-	if err != nil {
-		return err
-	}
-
-	if _, err := io.Copy(d, s); err != nil {
-		d.Close()
-		return err
-	}
-
-	return d.Close()
-}
-
-// FileExists returns true if the file at path exists.
-func FileExists(path string) bool {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return false
-	}
-
-	return true
-}
-
 // copyAll copies all files and directories from srcDir to dstDir recursively.
 // Note that this function does not support links.
 func copyAll(dstDir, srcDir string) error {
@@ -1746,3 +1680,26 @@ func (n *NetworkHarness) RestoreDb(hn *HarnessNode) error {
 
 	return nil
 }
+
+// getChanPointFundingTxid returns the given channel point's funding txid in
+// raw bytes.
+func getChanPointFundingTxid(chanPoint *lnrpc.ChannelPoint) ([]byte, error) {
+	var txid []byte
+
+	// A channel point's funding txid can be get/set as a byte slice or a
+	// string. In the case it is a string, decode it.
+	switch chanPoint.GetFundingTxid().(type) {
+	case *lnrpc.ChannelPoint_FundingTxidBytes:
+		txid = chanPoint.GetFundingTxidBytes()
+	case *lnrpc.ChannelPoint_FundingTxidStr:
+		s := chanPoint.GetFundingTxidStr()
+		h, err := chainhash.NewHashFromStr(s)
+		if err != nil {
+			return nil, err
+		}
+
+		txid = h[:]
+	}
+
+	return txid, nil
+}

lntest/itest/assertions.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
-	"github.com/btcsuite/btcd/integration/rpctest"
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
@@ -691,7 +690,7 @@ func assertChannelPolicy(t *harnessTest, node *lntest.HarnessNode,
 // assertMinerBlockHeightDelta ensures that tempMiner is 'delta' blocks ahead
 // of miner.
 func assertMinerBlockHeightDelta(t *harnessTest,
-	miner, tempMiner *rpctest.Harness, delta int32) {
+	miner, tempMiner *lntest.HarnessMiner, delta int32) {
 
 	// Ensure the chain lengths are what we expect.
 	var predErr error

lntest/itest/lnd_channel_force_close_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	"github.com/btcsuite/btcd/blockchain"
-	"github.com/btcsuite/btcd/integration/rpctest"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 	"github.com/go-errors/errors"
@@ -212,7 +211,7 @@ func testCommitmentTransactionDeadline(net *lntest.NetworkHarness,
 // calculateTxnsFeeRate takes a list of transactions and estimates the fee rate
 // used to sweep them.
 func calculateTxnsFeeRate(t *testing.T,
-	miner *rpctest.Harness, txns []*wire.MsgTx) int64 {
+	miner *lntest.HarnessMiner, txns []*wire.MsgTx) int64 {
 
 	var totalWeight, totalFee int64
 	for _, tx := range txns {

lntest/itest/lnd_open_channel_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/btcsuite/btcd/btcjson"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/integration/rpctest"
-	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/btcsuite/btcutil"
 	"github.com/lightningnetwork/lnd/funding"
 	"github.com/lightningnetwork/lnd/lnrpc"
@@ -34,16 +33,13 @@ func testOpenChannelAfterReorg(net *lntest.NetworkHarness, t *harnessTest) {
 	)
 
 	// Set up a new miner that we can use to cause a reorg.
-	tempLogDir := fmt.Sprintf("%s/.tempminerlogs", lntest.GetLogDir())
+	tempLogDir := ".tempminerlogs"
 	logFilename := "output-open_channel_reorg-temp_miner.log"
-	tempMiner, tempMinerCleanUp, err := lntest.NewMiner(
-		tempLogDir, logFilename, harnessNetParams,
-		&rpcclient.NotificationHandlers{}, lntest.GetBtcdBinary(),
-	)
+	tempMiner, err := lntest.NewTempMiner(tempLogDir, logFilename)
 	require.NoError(t.t, err, "failed to create temp miner")
 	defer func() {
 		require.NoError(
-			t.t, tempMinerCleanUp(),
+			t.t, tempMiner.Stop(),
 			"failed to clean up temp miner",
 		)
 	}()
@@ -61,7 +57,7 @@ func testOpenChannelAfterReorg(net *lntest.NetworkHarness, t *harnessTest) {
 	if err != nil {
 		t.Fatalf("unable to remove node: %v", err)
 	}
-	nodeSlice := []*rpctest.Harness{net.Miner, tempMiner}
+	nodeSlice := []*rpctest.Harness{net.Miner.Harness, tempMiner.Harness}
 	if err := rpctest.JoinNodes(nodeSlice, rpctest.Blocks); err != nil {
 		t.Fatalf("unable to join node on blocks: %v", err)
 	}
@@ -186,7 +182,7 @@ func testOpenChannelAfterReorg(net *lntest.NetworkHarness, t *harnessTest) {
 		t.Fatalf("unable to remove node: %v", err)
 	}
 
-	nodes := []*rpctest.Harness{tempMiner, net.Miner}
+	nodes := []*rpctest.Harness{tempMiner.Harness, net.Miner.Harness}
 	if err := rpctest.JoinNodes(nodes, rpctest.Blocks); err != nil {
 		t.Fatalf("unable to join node on blocks: %v", err)
 	}

lntest/itest/lnd_rpc_middleware_interceptor_test.go

@@ -403,9 +403,8 @@ func middlewareMandatoryTest(t *testing.T, node *lntest.HarnessNode,
 	// test case. So we need to do the wait and client setup manually here.
 	conn, err := node.ConnectRPC(true)
 	require.NoError(t, err)
-	err = node.WaitUntilStateReached(
-		conn, defaultTimeout, lnrpc.WalletState_RPC_ACTIVE,
-	)
+	node.InitRPCClients(conn)
+	err = node.WaitUntilStateReached(lnrpc.WalletState_RPC_ACTIVE)
 	require.NoError(t, err)
 	node.LightningClient = lnrpc.NewLightningClient(conn)
 

lntest/itest/lnd_test.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/btcsuite/btcd/integration/rpctest"
-	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/lightningnetwork/lnd/lntest"
 	"github.com/stretchr/testify/require"
 )
@@ -113,14 +112,10 @@ func TestLightningNetworkDaemon(t *testing.T) {
 	// guarantees of getting included in to blocks.
 	//
 	// We will also connect it to our chain backend.
-	minerLogDir := fmt.Sprintf("%s/.minerlogs", logDir)
-	miner, minerCleanUp, err := lntest.NewMiner(
-		minerLogDir, "output_btcd_miner.log", harnessNetParams,
-		&rpcclient.NotificationHandlers{}, lntest.GetBtcdBinary(),
-	)
+	miner, err := lntest.NewMiner()
 	require.NoError(t, err, "failed to create new miner")
 	defer func() {
-		require.NoError(t, minerCleanUp(), "failed to clean up miner")
+		require.NoError(t, miner.Stop(), "failed to stop miner")
 	}()
 
 	// Start a chain backend.

lntest/test_common.go

@@ -1,10 +1,16 @@
 package lntest
 
 import (
+	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"net"
+	"os"
 	"sync/atomic"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/lnrpc"
 )
 
 const (
@@ -114,3 +120,68 @@ func GenerateBtcdListenerAddresses() (string, string) {
 	return fmt.Sprintf(listenerFormat, NextAvailablePort()),
 		fmt.Sprintf(listenerFormat, NextAvailablePort())
 }
+
+// MakeOutpoint returns the outpoint of the channel's funding transaction.
+func MakeOutpoint(chanPoint *lnrpc.ChannelPoint) (wire.OutPoint, error) {
+	fundingTxID, err := lnrpc.GetChanPointFundingTxid(chanPoint)
+	if err != nil {
+		return wire.OutPoint{}, err
+	}
+
+	return wire.OutPoint{
+		Hash:  *fundingTxID,
+		Index: chanPoint.OutputIndex,
+	}, nil
+}
+
+// CheckChannelPolicy checks that the policy matches the expected one.
+func CheckChannelPolicy(policy, expectedPolicy *lnrpc.RoutingPolicy) error {
+	if policy.FeeBaseMsat != expectedPolicy.FeeBaseMsat {
+		return fmt.Errorf("expected base fee %v, got %v",
+			expectedPolicy.FeeBaseMsat, policy.FeeBaseMsat)
+	}
+	if policy.FeeRateMilliMsat != expectedPolicy.FeeRateMilliMsat {
+		return fmt.Errorf("expected fee rate %v, got %v",
+			expectedPolicy.FeeRateMilliMsat,
+			policy.FeeRateMilliMsat)
+	}
+	if policy.TimeLockDelta != expectedPolicy.TimeLockDelta {
+		return fmt.Errorf("expected time lock delta %v, got %v",
+			expectedPolicy.TimeLockDelta,
+			policy.TimeLockDelta)
+	}
+	if policy.MinHtlc != expectedPolicy.MinHtlc {
+		return fmt.Errorf("expected min htlc %v, got %v",
+			expectedPolicy.MinHtlc, policy.MinHtlc)
+	}
+	if policy.MaxHtlcMsat != expectedPolicy.MaxHtlcMsat {
+		return fmt.Errorf("expected max htlc %v, got %v",
+			expectedPolicy.MaxHtlcMsat, policy.MaxHtlcMsat)
+	}
+	if policy.Disabled != expectedPolicy.Disabled {
+		return errors.New("edge should be disabled but isn't")
+	}
+
+	return nil
+}
+
+// CopyFile copies the file src to dest.
+func CopyFile(dest, src string) error {
+	s, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	d, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(d, s); err != nil {
+		d.Close()
+		return err
+	}
+
+	return d.Close()
+}