Merge pull request #965 from cfromknecht/makefile-fix-and-timeout

Switch to Makefile Build System

.travis.yml

@@ -1,34 +1,19 @@
 language: go
 go:
-  - 1.9.4
+  - "1.9.4"
   - "1.10"
 sudo: required
 install:
   - sudo add-apt-repository -y ppa:bitcoin/bitcoin -y
   - sudo apt-get update -q
   - sudo apt-get install bitcoind -y
-  - GLIDE_TAG=v0.12.3
-  - GLIDE_DOWNLOAD="https://github.com/Masterminds/glide/releases/download/$GLIDE_TAG/glide-$GLIDE_TAG-linux-amd64.tar.gz"
-  - curl -L $GLIDE_DOWNLOAD | tar -xvz
   - export PATH=$PATH:$PWD/linux-amd64/
-  - BTCD_VERSION=$(cat Gopkg.toml | grep -A1 btcd | tail -n1 | awk '{ print $3 }' | tr -d '"')
-  - mkdir -p $GOPATH/src/github.com/roasbeef/
-  - pushd $GOPATH/src/github.com/roasbeef/
-  - git clone https://github.com/roasbeef/btcd
-  - pushd btcd
-  - git checkout $BTCD_VERSION
-  - glide install
-  - go install . ./cmd/...
-  - popd
-  - popd
 env:
-  global:
-    - LOGS=true
   matrix:
     - RACE=false
     - RACE=true
 script:
-  - export PATH=$PATH:$HOME/gopath/bin
+  - export PATH=$PATH:$GOPATH/bin
-  - ./gotest.sh
+  - make travis
 after_script:
   - find *.log | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"

Makefile

@@ -1,92 +1,241 @@
 PKG := github.com/lightningnetwork/lnd
-HAVE_DEP := $(shell command -v dep 2> /dev/null)
+ESCPKG := github.com\/lightningnetwork\/lnd
+
+DEP_PKG := github.com/golang/dep/cmd/dep
+BTCD_PKG := github.com/roasbeef/btcd
+GLIDE_PKG := github.com/Masterminds/glide
+GOVERALLS_PKG := github.com/mattn/goveralls
+LINT_PKG := gopkg.in/alecthomas/gometalinter.v1
+
+GO_BIN := ${GOPATH}/bin
+DEP_BIN := $(GO_BIN)/dep
+BTCD_BIN := $(GO_BIN)/btcd
+GLIDE_BIN := $(GO_BIN)/glide
+GOVERALLS_BIN := $(GO_BIN)/goveralls
+LINT_BIN := $(GO_BIN)/gometalinter.v1
+
+HAVE_DEP := $(shell command -v $(DEP_BIN) 2> /dev/null)
+HAVE_BTCD := $(shell command -v $(BTCD_BIN) 2> /dev/null)
+HAVE_GLIDE := $(shell command -v $(GLIDE_BIN) 2> /dev/null)
+HAVE_GOVERALLS := $(shell command -v $(GOVERALLS_BIN) 2> /dev/null)
+HAVE_LINTER := $(shell command -v $(LINT_BIN) 2> /dev/null)
+
+BTCD_DIR :=${GOPATH}/src/$(BTCD_PKG)
+
 COMMIT := $(shell git rev-parse HEAD)
 LDFLAGS := -ldflags "-X main.Commit=$(COMMIT)"
-RM := rm
 
-all: scratch
+GLIDE_COMMIT := 84607742b10f492430762d038e954236bbaf23f7
+BTCD_COMMIT := $(shell cat Gopkg.toml | \
+		grep -A1 $(BTCD_PKG) | \
+		tail -n1 | \
+		awk '{ print $$3 }' | \
+		tr -d '"')
 
+GOBUILD := go build -v
+GOINSTALL := go install -v
+GOTEST := go test -v
 
-# ==============================================================================
+GOLIST := go list $(PKG)/... | grep -v '/vendor/'
-# INSTALLATION
+GOLISTCOVER := $(shell go list -f '{{.ImportPath}}' ./... | sed -e 's/^$(ESCPKG)/./')
-# ==============================================================================
+GOLISTLINT := $(shell go list -f '{{.Dir}}' ./... | grep -v 'lnrpc')
 
-deps:
+COVER = for dir in $(GOLISTCOVER); do \
-ifndef HAVE_DEP
+		$(GOTEST) $(TEST_FLAGS) \
-	@echo "Fetching dep."
+			-covermode=count \
-	go get -u github.com/golang/dep/cmd/dep
+			-coverprofile=$$dir/profile.tmp $$dir; \
-endif
+		\
-	@echo "Building dependencies."
+		if [ -f $$dir/profile.tmp ]; then \
+			cat $$dir/profile.tmp | \
+				tail -n +2 >> profile.cov; \
+			$(RM) $$dir/profile.tmp; \
+		fi \
+	done
+
+LINT = $(LINT_BIN) \
+	--disable-all \
+	--enable=gofmt \
+	--enable=vet \
+	--enable=golint \
+	--line-length=72 \
+	--deadline=4m $(GOLISTLINT) 2>&1 | \
+	grep -v 'ALL_CAPS\|OP_' 2>&1 | \
+	tee /dev/stderr
+
+CGO_STATUS_QUO := ${CGO_ENABLED}
+
+RM := rm -f
+CP := cp
+MAKE := make
+XARGS := xargs -L 1
+
+GREEN := "\\033[0;32m"
+NC := "\\033[0m"
+define print
+	echo $(GREEN)$1$(NC)
+endef
+
+include make/testing_flags.mk
+
+default: scratch
+
+all: scratch check install
+
+# ============
+# DEPENDENCIES
+# ============
+
+$(DEP_BIN):
+	@$(call print, "Fetching dep.")
+	go get -u $(DEP_PKG)
+
+$(GLIDE_BIN):
+	@$(call print, "Fetching glide.")
+	go get -d $(GLIDE_PKG)
+	cd ${GOPATH}/src/$(GLIDE_PKG) && git checkout $(GLIDE_COMMIT) 
+	$(GOINSTALL) $(GLIDE_PKG)
+
+$(GOVERALLS_BIN):
+	@$(call print, "Fetching goveralls.")
+	go get -u $(GOVERALLS_PKG)
+
+$(LINT_BIN):
+	@$(call print, "Fetching gometalinter.v1")
+	go get -u $(LINT_PKG)
+	$(GOINSTALL) $(LINT_PKG)
+
+dep: $(DEP_BIN)
+	@$(call print, "Compiling dependencies.")
 	dep ensure -v
 
-build: 
+$(BTCD_DIR):
-	@echo "Building lnd and lncli."
+	@$(call print, "Fetching btcd.")
-	go build -v -o lnd $(LDFLAGS) $(PKG)
+	go get -d github.com/roasbeef/btcd
-	go build -v -o lncli $(LDFLAGS) $(PKG)/cmd/lncli
+
+btcd: $(GLIDE_BIN) $(BTCD_DIR)
+	@$(call print, "Compiling btcd dependencies.")
+	cd $(BTCD_DIR) && git checkout $(BTCD_COMMIT) && glide install
+	@$(call print, "Installing btcd and btcctl.")
+	$(GOINSTALL) $(BTCD_PKG)
+	$(GOINSTALL) $(BTCD_PKG)/cmd/btcctl
+
+# ============
+# INSTALLATION
+# ============
+
+build:
+	@$(call print, "Building lnd and lncli.")
+	$(GOBUILD) -o lnd $(LDFLAGS) $(PKG)
+	$(GOBUILD) -o lncli $(LDFLAGS) $(PKG)/cmd/lncli
 
 install:
-	@echo "Installing lnd and lncli."
+	@$(call print, "Installing lnd and lncli.")
-	go install -v $(LDFLAGS) $(PKG)
+	$(CP) ./lnd $(LND_BIN)
-	go install -v $(LDFLAGS) $(PKG)/cmd/lncli
+	$(CP) ./lncli $(LNCLI_BIN)
 
-scratch: deps build
+scratch: dep build
 
 
-# ==============================================================================
+# =======
 # TESTING
-# ==============================================================================
+# =======
-
-# Define the integration test.run filter if the icase argument was provided.
-ifneq ($(icase),)
-	ITESTCASE := -test.run=TestLightningNetworkDaemon/$(icase)
-endif
-
-# UNIT_TARGTED is defined iff a specific package and/or unit test case is being
-# targeted.
-UNIT_TARGETED = 
-
-# If specific package is being unit tested, construct the full name of the
-# subpackage.
-ifneq ($(pkg),)
-	UNITPKG := $(PKG)/$(pkg)
-	UNIT_TARGETED = yes
-endif
-
-# If a specific unit test case is being target, construct test.run filter.
-ifneq ($(case),)
-	UNITCASE := -test.run=$(case)
-	UNIT_TARGETED = yes
-endif
-
-# If no unit targeting was input, default to running all tests. Otherwise,
-# construct the command to run the specific package/test case.
-ifndef UNIT_TARGETED
-	UNIT := go list $(PKG)/... | grep -v '/vendor/' | xargs go test
-else
-	UNIT := go test -test.v $(UNITPKG) $(UNITCASE)
-endif
 
 check: unit itest
 
-itest: install
+itest: btcd build
-	@echo "Running integration tests."
+	@$(call print, "Running integration tests.")
-	go test -v -tags rpctest -logoutput $(ITESTCASE)
+	$(ITEST)
 
-unit:
+unit: btcd
-	@echo "Running unit tests."
+	@$(call print, "Running unit tests.")
 	$(UNIT)
 
+unit-cover:
+	@$(call print, "Running unit coverage tests.")
+	echo "mode: count" > profile.cov
+	$(COVER)
+		
+unit-race:
+	@$(call print, "Running unit race tests.")
+	export CGO_ENABLED=1; env GORACE="history_size=7 halt_on_errors=1" $(UNIT_RACE)
+	export CGO_ENABLED=$(CGO_STATUS_QUO)
 
-# ==============================================================================
+# =============
+# FLAKE HUNTING
+# =============
+
+flakehunter: build
+	@$(call print, "Flake hunting integration tests.")
+	$(ITEST)
+	while [ $$? -eq 0 ]; do /bin/sh -c "$(ITEST)"; done
+
+flake-unit:
+	@$(call print, "Flake hunting unit tests.")
+	$(UNIT) -count=1
+	while [ $$? -eq 0 ]; do /bin/sh -c "$(UNIT) -count=1"; done
+
+# ======
+# TRAVIS
+# ======
+
+ifeq ($(RACE), false)
+travis: lint scratch itest unit-cover $(GOVERALLS_BIN)
+	@$(call print, "Sending coverage report.")
+	$(GOVERALLS_BIN) -coverprofile=profile.cov -service=travis-ci
+endif
+
+ifeq ($(RACE), true)
+travis: lint dep btcd unit-race
+endif
+
+
+# =========
 # UTILITIES
-# ==============================================================================
+# =========
 
 fmt:
-	go list $(PKG)/... | grep -v '/vendor/' | xargs go fmt -x
+	@$(call print, "Formatting source.")
+	$(GOLIST) | $(XARGS) go fmt -x
+
+lint: $(LINT_BIN)
+	@$(call print, "Linting source.")
+	$(LINT_BIN) --install 1> /dev/null
+	test -z "$($(LINT))"
+
+list:
+	@$(call print, "Listing commands.")
+	@$(MAKE) -qp | \
+		awk -F':' '/^[a-zA-Z0-9][^$$#\/\t=]*:([^=]|$$)/ {split($$1,A,/ /);for(i in A)print A[i]}' | \
+		grep -v Makefile | \
+		sort
+
+rpc:
+	@$(call print, "Compiling protos.")
+	cd ./lnrpc; ./gen_protos.sh
 
 clean:
+	@$(call print, "Cleaning source.$(NC)")
 	$(RM) ./lnd ./lncli
-	$(RM) -rf vendor
+	$(RM) -r ./vendor
 
 
-.PHONY: all deps build install scratch check itest unit fmt clean
+.PHONY: all \
+	btcd\
+	default \
+	dep \
+	build \
+	install \
+	scratch \
+	check \
+	itest \
+	unit \
+	unit-cover \
+	unit-race \
+	flakehunter \
+	flake-unit \
+	travis \
+	fmt \
+	lint \
+	list \
+	rpc \
+	clean

gotest.sh

@@ -1,179 +0,0 @@
-#!/bin/bash
-# The script does automatic checking on a Go package and its sub-packages, including:
-# 1. gofmt         (http://golang.org/cmd/gofmt/)
-# 2. golint        (https://github.com/golang/lint)
-# 3. go vet        (http://golang.org/cmd/vet)
-# 4. gosimple      (https://github.com/dominikh/go-simple)
-# 5. unconvert     (https://github.com/mdempsky/unconvert)
-# 6. race detector (http://blog.golang.org/race-detector)
-# 7. test coverage (http://blog.golang.org/cover)
-#
-# gometalinter (github.com/alecthomas/gometalinter) is used to run each static
-# checker.
-
-declare GREEN='\033[0;32m'
-declare NC='\033[0m'
-print () {
-    echo -e "${GREEN}$1${NC}"
-}
-
-# check_test_ports checks that test lnd ports are not used.
-check_test_ports() {
-    # Make sure that test lnd ports are not used.
-    o=$(lsof -i :19555,19556 | sed '1d' | awk '{ printf "%s\n", $2 }')
-    if [ "$o" != "" ]; then
-        printf "Can't run the lnd tests:\n"
-        printf "some program is using the test lnd ports (19555 | 19556)\n"
-        exit 1
-    fi
-}
-
-# test_with_profile run test coverage on each subdirectories and merge the
-# coverage profile. Be aware that we are skipping the integration tests, as the
-# tool won't gather any useful coverage information from them.
-test_with_coverage_profile() {
-    print "* Running tests with creating coverage profile"
-
-    echo "mode: count" > profile.cov
-
-    # Standard go tooling behavior is to ignore dirs with leading underscores.
-    for dir in $(find . -maxdepth 10 \
-    -not -path './.git*' \
-    -not -path '*/_*' \
-    -not -path '.' \
-    -not -path './cmd*' \
-    -not -path './release*' \
-    -not -path './vendor*' \
-    -type d)
-    do
-    if ls $dir/*.go &> /dev/null; then
-      go test -v -covermode=count -coverprofile=$dir/profile.tmp $dir
-
-      if [ -f $dir/profile.tmp ]; then
-        cat $dir/profile.tmp | tail -n +2 >> profile.cov
-        rm $dir/profile.tmp
-      fi
-    fi
-    done
-
-    if [ "$TRAVIS" == "true" ] ; then
-        print "* Send test coverage to travis server"
-        # Make sure goveralls is installed and $GOPATH/bin is in your path.
-        if [ ! -x "$(type -p goveralls)" ]; then
-            print "** Install goveralls:"
-            go get github.com/mattn/goveralls
-        fi
-
-        print "** Submit the test coverage result to coveralls.io"
-        goveralls -coverprofile=profile.cov -service=travis-ci
-    fi
-}
-
-# test_race_conditions run standard go test without creating coverage
-# profile but with race condition checks.
-test_race_conditions() {
-    print "* Running tests with the race condition detector"
-    
-    test_targets=$(go list ./... | grep -v '/vendor/')
-    env GORACE="history_size=7 halt_on_error=1" go test -v -p 1 -race $test_targets
-}
-
-# lint_check runs static checks.
-lint_check() {
-    print "* Run static checks"
-
-    # Make sure gometalinter is installed and $GOPATH/bin is in your path.
-    if [ ! -x "$(type -p gometalinter.v1)" ]; then
-        print "** Install gometalinter"
-        go get -u gopkg.in/alecthomas/gometalinter.v1
-        gometalinter.v1 --install
-    fi
-
-    # Update metalinter if needed.
-    gometalinter.v1 --install 1>/dev/null
-
-    # Automatic checks
-    linter_targets=$(go list -f '{{.Dir}}' ./... | grep -v lnrpc)
-    test -z "$(gometalinter.v1 --disable-all \
-    --enable=gofmt \
-    --enable=vet \
-    --enable=golint \
-    --line-length=72 \
-    --deadline=4m $linter_targets 2>&1 | grep -v 'ALL_CAPS\|OP_' 2>&1 | tee /dev/stderr)"
-}
-
-set -e
-
-print "====+ Start +===="
-
-# Read input flags and initialize variables
-NEED_LINT="false"
-NEED_COVERAGE="false"
-NEED_RACE="false"
-NEED_INSTALL="false"
-NEED_LOGS="false"
-
-while getopts "lrcio" flag; do
-        case "${flag}" in
-            l) NEED_LINT="true" ;;
-            r) NEED_RACE="true" ;;
-            c) NEED_COVERAGE="true" ;;
-            i) NEED_INSTALL="true" ;;
-            o) NEED_LOGS="true" ;;
-            *)
-                printf '\nUsage: %s [-l] [-r] [-c] [-i] [-o], where:\n' $0
-                printf ' -l: include code lint check\n'
-                printf ' -r: run tests with race condition check\n'
-                printf ' -c: run tests with test coverage\n'
-                printf ' -i: reinstall project dependencies\n'
-                printf ' -o: generate logs for spawned lnd instances\n'
-                exit 1 ;;
-        esac
-    done
-
-# remove the options from the positional parameters
-shift $(( OPTIND - 1 ))
-
-# Make sure dep is installed and $GOPATH/bin is in your path.
-if [ ! -x "$(type -p dep)" ]; then
-    print "* Install dep"
-    go get -u github.com/golang/dep/cmd/dep
-fi
-
-# Install the dependency if vendor directory not exist or if flag have been
-# specified.
-if [ "$NEED_INSTALL" == "true" ] || [ ! -d "./vendor" ]; then
-    print "* Install dependencies"
-    dep ensure
-fi
-
-# Required for the integration tests
-print "* Build source"
-go install -v . ./cmd/...
-
-# Lint check is first because we shouldn't run tests on garbage code.
-if [ "$NEED_LINT" == "true" ] || [ "$RACE" == "false" ]; then
-    lint_check
-fi
-
-# Race condition second because we shouldn't check coverage on buggy code.
-if [ "$NEED_RACE" == "true" ] || [ "$RACE" == "true" ]; then
-    test_race_conditions
-fi
-
-# Test coverage is third because in this case code should work properly and
-# we may calmly send coverage profile (if script is run on travis)
-if [ "$NEED_COVERAGE" == "true" ] || [ "$RACE" == "false" ]; then
-    print "* Running integration tests"
-
-    LOGOUTPUT_FLAG=""
-    if [ "$NEED_LOGS" == "true" ] || [ "$LOGS" == "true" ]; then
-        LOGOUTPUT_FLAG="-logoutput=true"
-    fi
-
-    go test -v -tags rpctest "$LOGOUTPUT_FLAG"
-
-    test_with_coverage_profile
-fi
-
-print "=====+ End +====="

lntest/node.go

@@ -245,7 +245,7 @@ func (hn *HarnessNode) start(lndError chan<- error) error {
 
 	args := hn.cfg.genArgs()
 	args = append(args, fmt.Sprintf("--profile=%d", 9000+hn.NodeID))
-	hn.cmd = exec.Command("lnd", args...)
+	hn.cmd = exec.Command("./lnd", args...)
 
 	// Redirect stderr output to buffer
 	var errb bytes.Buffer

make/testing_flags.mk

@@ -0,0 +1,45 @@
+TEST_FLAGS =
+
+# If specific package is being unit tested, construct the full name of the
+# subpackage.
+ifneq ($(pkg),)
+UNITPKG := $(PKG)/$(pkg)
+UNIT_TARGETED = yes
+endif
+
+# If a specific unit test case is being target, construct test.run filter.
+ifneq ($(case),)
+TEST_FLAGS += -test.run=$(case)
+UNIT_TARGETED = yes
+endif
+
+# Define the integration test.run filter if the icase argument was provided.
+ifneq ($(icase),)
+TEST_FLAGS += -test.run=TestLightningNetworkDaemon/$(icase)
+endif
+
+# If a timeout was requested, construct initialize the proper flag for the go
+# test command.
+ifneq ($(timeout),)
+TEST_FLAGS += -test.timeout=$(timeout)
+endif
+
+# UNIT_TARGTED is undefined iff a specific package and/or unit test case is
+# not being targeted.
+UNIT_TARGETED ?= no
+
+# If a specific package/test case was requested, run the unit test for the
+# targeted case. Otherwise, default to running all tests.
+ifeq ($(UNIT_TARGETED), yes)
+UNIT := $(GOTEST) $(TEST_FLAGS) $(UNITPKG)
+UNIT_RACE := $(GOTEST) $(TEST_FLAGS) -race $(UNITPKG)
+endif
+
+ifeq ($(UNIT_TARGETED), no)
+UNIT := $(GOLIST) | $(XARGS) $(GOTEST) $(TEST_FLAGS) 
+UNIT_RACE := $(UNIT) -race 
+endif
+
+
+# Construct the integration test command with the added build flags.
+ITEST := $(GOTEST) $(TEST_FLAGS) -tags rpctest -logoutput