itest: shuffle test cases to even out blocks mined in tranches

This commit adds a new flag to shuffle all the test cases before running
them so tests which require lots of blocks to be mined are less likely
to be run in the same tranch.

The other benefit is this approach provides a more efficient way to
figure which tests are broken since all the differnet backends are
running different tranches in their builds, we can identify more failed
tests in one push.

Makefile

@@ -220,7 +220,7 @@ clean-itest-logs:
 itest-only: clean-itest-logs db-instance
 	@$(call print, "Running integration tests with ${backend} backend.")
 	date
-	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS) -test.v
+	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) -test.v
 	$(COLLECT_ITEST_COVERAGE)
 
 #? itest: Build and run integration tests
@@ -233,7 +233,7 @@ itest-race: build-itest-race itest-only
 itest-parallel: clean-itest-logs build-itest db-instance
 	@$(call print, "Running tests")
 	date
-	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)
+	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
 	$(COLLECT_ITEST_COVERAGE)
 
 #? itest-clean: Kill all running itest processes

itest/list_on_test.go

@@ -2,7 +2,9 @@
 
 package itest
 
-import "github.com/lightningnetwork/lnd/lntest"
+import (
+	"github.com/lightningnetwork/lnd/lntest"
+)
 
 var allTestCases = []*lntest.TestCase{
 	{

itest/lnd_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/lightningnetwork/lnd/lntest/port"
 	"github.com/lightningnetwork/lnd/lntest/wait"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/rand"
 	"google.golang.org/grpc/grpclog"
 )
 
@@ -61,6 +62,13 @@ var (
 			"0-based index specified by the -runtranche flag",
 	)
 
+	// shuffleSeedFlag is the source of randomness used to shuffle the test
+	// cases. If not specified, the test cases won't be shuffled.
+	shuffleSeedFlag = flag.Uint64(
+		"shuffleseed", 0, "if set, shuffles the test cases using this "+
+			"as the source of randomness",
+	)
+
 	// testCasesRunTranche is the 0-based index of the split test cases
 	// tranche to run in the current invocation.
 	testCasesRunTranche = flag.Uint(
@@ -160,6 +168,32 @@ func TestLightningNetworkDaemon(t *testing.T) {
 		harnessTest.CurrentHeight()-height)
 }
 
+// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is
+// set and not 0. In parallel tests we want to shuffle the test cases so they
+// are executed in a random order. This is done to even out the blocks mined in
+// each test tranche so they can run faster.
+//
+// NOTE: Because the parallel tests are initialized with the same seed (job
+// ID), they will always have the same order.
+func maybeShuffleTestCases() {
+	// Exit if not set.
+	if shuffleSeedFlag == nil {
+		return
+	}
+
+	// Exit if set to 0.
+	if *shuffleSeedFlag == 0 {
+		return
+	}
+
+	// Init the seed and shuffle the test cases.
+	rand.Seed(*shuffleSeedFlag)
+	rand.Shuffle(len(allTestCases), func(i, j int) {
+		allTestCases[i], allTestCases[j] =
+			allTestCases[j], allTestCases[i]
+	})
+}
+
 // getTestCaseSplitTranche returns the sub slice of the test cases that should
 // be run as the current split tranche as well as the index and slice offset of
 // the tranche.
@@ -182,6 +216,9 @@ func getTestCaseSplitTranche() ([]*lntest.TestCase, uint, uint) {
 		runTranche = 0
 	}
 
+	// Shuffle the test cases if the `shuffleseed` flag is set.
+	maybeShuffleTestCases()
+
 	numCases := uint(len(allTestCases))
 	testsPerTranche := numCases / numTranches
 	trancheOffset := runTranche * testsPerTranche

make/testing_flags.mk

@@ -10,6 +10,7 @@ COVER_PKG = $$(go list -deps -tags="$(DEV_TAGS)" ./... | grep '$(PKG)' | grep -v
 NUM_ITEST_TRANCHES = 4
 ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
 POSTGRES_START_DELAY = 5
+SHUFFLE_SEED = 0
 
 # If rpc option is set also add all extra RPC tags to DEV_TAGS
 ifneq ($(with-rpc),)
@@ -27,6 +28,11 @@ ifneq ($(parallel),)
 ITEST_PARALLELISM = $(parallel)
 endif
 
+# Set the seed for shuffling the test cases.
+ifneq ($(shuffleseed),)
+SHUFFLE_SEED = $(shuffleseed)
+endif
+
 # Windows needs to append a .exe suffix to all executable files, otherwise it
 # won't run them.
 ifneq ($(windows),)

scripts/itest_parallel.sh

@@ -3,9 +3,10 @@
 # Get all the variables.
 PROCESSES=$1
 TRANCHES=$2
+SHUFFLE_SEED=$3
 
-# Here we also shift 2 times and get the rest of our flags to pass on in $@.
-shift 2
+# Here we also shift 3 times and get the rest of our flags to pass on in $@.
+shift 3
 
 # Create a variable to hold the final exit code.
 exit_code=0
@@ -13,7 +14,7 @@ exit_code=0
 # Run commands using xargs in parallel and capture their PIDs
 pids=()
 for ((i=0; i<PROCESSES; i++)); do 
-    scripts/itest_part.sh $i $TRANCHES $@ &
+    scripts/itest_part.sh $i $TRANCHES $SHUFFLE_SEED $@ &
     pids+=($!)
 done
 

scripts/itest_part.sh

@@ -5,11 +5,11 @@ WORKDIR=$(pwd)/itest
 
 TRANCHE=$1
 NUM_TRANCHES=$2
+SHUFFLE_SEED_PARAM=$3
 
-# Shift the passed parameters by two, giving us all remaining testing flags in
+# Shift the passed parameters by three, giving us all remaining testing flags in
 # the $@ special variable.
-shift
-shift
+shift 3
 
 # Windows insists on having the .exe suffix for an executable, we need to add
 # that here if necessary.
@@ -18,9 +18,9 @@ LND_EXEC="$WORKDIR"/lnd-itest"$EXEC_SUFFIX"
 BTCD_EXEC="$WORKDIR"/btcd-itest"$EXEC_SUFFIX"
 export GOCOVERDIR="$WORKDIR/cover"
 mkdir -p "$GOCOVERDIR"
-echo $EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE
+echo $EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM
 
 # Exit code 255 causes the parallel jobs to abort, so if one part fails the
 # other is aborted too.
 cd "$WORKDIR" || exit 255
-$EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE || exit 255
+$EXEC "$@" -logoutput -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM || exit 255