diff --git a/.travis.yml b/.travis.yml index 911042262..1b17e16a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,19 @@ language: go go: - - 1.9.4 + - "1.9.4" - "1.10" sudo: required install: - sudo add-apt-repository -y ppa:bitcoin/bitcoin -y - sudo apt-get update -q - sudo apt-get install bitcoind -y - - GLIDE_TAG=v0.12.3 - - GLIDE_DOWNLOAD="https://github.com/Masterminds/glide/releases/download/$GLIDE_TAG/glide-$GLIDE_TAG-linux-amd64.tar.gz" - - curl -L $GLIDE_DOWNLOAD | tar -xvz - export PATH=$PATH:$PWD/linux-amd64/ - - BTCD_VERSION=$(cat Gopkg.toml | grep -A1 btcd | tail -n1 | awk '{ print $3 }' | tr -d '"') - - mkdir -p $GOPATH/src/github.com/roasbeef/ - - pushd $GOPATH/src/github.com/roasbeef/ - - git clone https://github.com/roasbeef/btcd - - pushd btcd - - git checkout $BTCD_VERSION - - glide install - - go install . ./cmd/... - - popd - - popd env: - global: - - LOGS=true matrix: - RACE=false - RACE=true script: - - export PATH=$PATH:$HOME/gopath/bin - - ./gotest.sh + - export PATH=$PATH:$GOPATH/bin + - make travis after_script: - find *.log | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'" diff --git a/Makefile b/Makefile index d39c21b46..e2c272923 100644 --- a/Makefile +++ b/Makefile @@ -1,92 +1,241 @@ PKG := github.com/lightningnetwork/lnd -HAVE_DEP := $(shell command -v dep 2> /dev/null) +ESCPKG := github.com\/lightningnetwork\/lnd + +DEP_PKG := github.com/golang/dep/cmd/dep +BTCD_PKG := github.com/roasbeef/btcd +GLIDE_PKG := github.com/Masterminds/glide +GOVERALLS_PKG := github.com/mattn/goveralls +LINT_PKG := gopkg.in/alecthomas/gometalinter.v1 + +GO_BIN := ${GOPATH}/bin +DEP_BIN := $(GO_BIN)/dep +BTCD_BIN := $(GO_BIN)/btcd +GLIDE_BIN := $(GO_BIN)/glide +GOVERALLS_BIN := $(GO_BIN)/goveralls +LINT_BIN := $(GO_BIN)/gometalinter.v1 + +HAVE_DEP := $(shell command -v $(DEP_BIN) 2> /dev/null) +HAVE_BTCD := $(shell command -v $(BTCD_BIN) 2> /dev/null) +HAVE_GLIDE := $(shell command -v $(GLIDE_BIN) 2> /dev/null) +HAVE_GOVERALLS := $(shell command -v $(GOVERALLS_BIN) 2> /dev/null) +HAVE_LINTER := $(shell command -v $(LINT_BIN) 2> /dev/null) + +BTCD_DIR :=${GOPATH}/src/$(BTCD_PKG) + COMMIT := $(shell git rev-parse HEAD) LDFLAGS := -ldflags "-X main.Commit=$(COMMIT)" -RM := rm -all: scratch +GLIDE_COMMIT := 84607742b10f492430762d038e954236bbaf23f7 +BTCD_COMMIT := $(shell cat Gopkg.toml | \ + grep -A1 $(BTCD_PKG) | \ + tail -n1 | \ + awk '{ print $$3 }' | \ + tr -d '"') +GOBUILD := go build -v +GOINSTALL := go install -v +GOTEST := go test -v -# ============================================================================== -# INSTALLATION -# ============================================================================== +GOLIST := go list $(PKG)/... | grep -v '/vendor/' +GOLISTCOVER := $(shell go list -f '{{.ImportPath}}' ./... | sed -e 's/^$(ESCPKG)/./') +GOLISTLINT := $(shell go list -f '{{.Dir}}' ./... | grep -v 'lnrpc') -deps: -ifndef HAVE_DEP - @echo "Fetching dep." - go get -u github.com/golang/dep/cmd/dep -endif - @echo "Building dependencies." +COVER = for dir in $(GOLISTCOVER); do \ + $(GOTEST) $(TEST_FLAGS) \ + -covermode=count \ + -coverprofile=$$dir/profile.tmp $$dir; \ + \ + if [ -f $$dir/profile.tmp ]; then \ + cat $$dir/profile.tmp | \ + tail -n +2 >> profile.cov; \ + $(RM) $$dir/profile.tmp; \ + fi \ + done + +LINT = $(LINT_BIN) \ + --disable-all \ + --enable=gofmt \ + --enable=vet \ + --enable=golint \ + --line-length=72 \ + --deadline=4m $(GOLISTLINT) 2>&1 | \ + grep -v 'ALL_CAPS\|OP_' 2>&1 | \ + tee /dev/stderr + +CGO_STATUS_QUO := ${CGO_ENABLED} + +RM := rm -f +CP := cp +MAKE := make +XARGS := xargs -L 1 + +GREEN := "\\033[0;32m" +NC := "\\033[0m" +define print + echo $(GREEN)$1$(NC) +endef + +include make/testing_flags.mk + +default: scratch + +all: scratch check install + +# ============ +# DEPENDENCIES +# ============ + +$(DEP_BIN): + @$(call print, "Fetching dep.") + go get -u $(DEP_PKG) + +$(GLIDE_BIN): + @$(call print, "Fetching glide.") + go get -d $(GLIDE_PKG) + cd ${GOPATH}/src/$(GLIDE_PKG) && git checkout $(GLIDE_COMMIT) + $(GOINSTALL) $(GLIDE_PKG) + +$(GOVERALLS_BIN): + @$(call print, "Fetching goveralls.") + go get -u $(GOVERALLS_PKG) + +$(LINT_BIN): + @$(call print, "Fetching gometalinter.v1") + go get -u $(LINT_PKG) + $(GOINSTALL) $(LINT_PKG) + +dep: $(DEP_BIN) + @$(call print, "Compiling dependencies.") dep ensure -v -build: - @echo "Building lnd and lncli." - go build -v -o lnd $(LDFLAGS) $(PKG) - go build -v -o lncli $(LDFLAGS) $(PKG)/cmd/lncli +$(BTCD_DIR): + @$(call print, "Fetching btcd.") + go get -d github.com/roasbeef/btcd + +btcd: $(GLIDE_BIN) $(BTCD_DIR) + @$(call print, "Compiling btcd dependencies.") + cd $(BTCD_DIR) && git checkout $(BTCD_COMMIT) && glide install + @$(call print, "Installing btcd and btcctl.") + $(GOINSTALL) $(BTCD_PKG) + $(GOINSTALL) $(BTCD_PKG)/cmd/btcctl + +# ============ +# INSTALLATION +# ============ + +build: + @$(call print, "Building lnd and lncli.") + $(GOBUILD) -o lnd $(LDFLAGS) $(PKG) + $(GOBUILD) -o lncli $(LDFLAGS) $(PKG)/cmd/lncli install: - @echo "Installing lnd and lncli." - go install -v $(LDFLAGS) $(PKG) - go install -v $(LDFLAGS) $(PKG)/cmd/lncli + @$(call print, "Installing lnd and lncli.") + $(CP) ./lnd $(LND_BIN) + $(CP) ./lncli $(LNCLI_BIN) -scratch: deps build +scratch: dep build -# ============================================================================== +# ======= # TESTING -# ============================================================================== - -# Define the integration test.run filter if the icase argument was provided. -ifneq ($(icase),) - ITESTCASE := -test.run=TestLightningNetworkDaemon/$(icase) -endif - -# UNIT_TARGTED is defined iff a specific package and/or unit test case is being -# targeted. -UNIT_TARGETED = - -# If specific package is being unit tested, construct the full name of the -# subpackage. -ifneq ($(pkg),) - UNITPKG := $(PKG)/$(pkg) - UNIT_TARGETED = yes -endif - -# If a specific unit test case is being target, construct test.run filter. -ifneq ($(case),) - UNITCASE := -test.run=$(case) - UNIT_TARGETED = yes -endif - -# If no unit targeting was input, default to running all tests. Otherwise, -# construct the command to run the specific package/test case. -ifndef UNIT_TARGETED - UNIT := go list $(PKG)/... | grep -v '/vendor/' | xargs go test -else - UNIT := go test -test.v $(UNITPKG) $(UNITCASE) -endif +# ======= check: unit itest -itest: install - @echo "Running integration tests." - go test -v -tags rpctest -logoutput $(ITESTCASE) +itest: btcd build + @$(call print, "Running integration tests.") + $(ITEST) -unit: - @echo "Running unit tests." +unit: btcd + @$(call print, "Running unit tests.") $(UNIT) +unit-cover: + @$(call print, "Running unit coverage tests.") + echo "mode: count" > profile.cov + $(COVER) + +unit-race: + @$(call print, "Running unit race tests.") + export CGO_ENABLED=1; env GORACE="history_size=7 halt_on_errors=1" $(UNIT_RACE) + export CGO_ENABLED=$(CGO_STATUS_QUO) -# ============================================================================== +# ============= +# FLAKE HUNTING +# ============= + +flakehunter: build + @$(call print, "Flake hunting integration tests.") + $(ITEST) + while [ $$? -eq 0 ]; do /bin/sh -c "$(ITEST)"; done + +flake-unit: + @$(call print, "Flake hunting unit tests.") + $(UNIT) -count=1 + while [ $$? -eq 0 ]; do /bin/sh -c "$(UNIT) -count=1"; done + +# ====== +# TRAVIS +# ====== + +ifeq ($(RACE), false) +travis: lint scratch itest unit-cover $(GOVERALLS_BIN) + @$(call print, "Sending coverage report.") + $(GOVERALLS_BIN) -coverprofile=profile.cov -service=travis-ci +endif + +ifeq ($(RACE), true) +travis: lint dep btcd unit-race +endif + + +# ========= # UTILITIES -# ============================================================================== +# ========= fmt: - go list $(PKG)/... | grep -v '/vendor/' | xargs go fmt -x + @$(call print, "Formatting source.") + $(GOLIST) | $(XARGS) go fmt -x + +lint: $(LINT_BIN) + @$(call print, "Linting source.") + $(LINT_BIN) --install 1> /dev/null + test -z "$($(LINT))" + +list: + @$(call print, "Listing commands.") + @$(MAKE) -qp | \ + awk -F':' '/^[a-zA-Z0-9][^$$#\/\t=]*:([^=]|$$)/ {split($$1,A,/ /);for(i in A)print A[i]}' | \ + grep -v Makefile | \ + sort + +rpc: + @$(call print, "Compiling protos.") + cd ./lnrpc; ./gen_protos.sh clean: + @$(call print, "Cleaning source.$(NC)") $(RM) ./lnd ./lncli - $(RM) -rf vendor + $(RM) -r ./vendor -.PHONY: all deps build install scratch check itest unit fmt clean +.PHONY: all \ + btcd\ + default \ + dep \ + build \ + install \ + scratch \ + check \ + itest \ + unit \ + unit-cover \ + unit-race \ + flakehunter \ + flake-unit \ + travis \ + fmt \ + lint \ + list \ + rpc \ + clean diff --git a/gotest.sh b/gotest.sh deleted file mode 100755 index a1d0b8b0e..000000000 --- a/gotest.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash -# The script does automatic checking on a Go package and its sub-packages, including: -# 1. gofmt (http://golang.org/cmd/gofmt/) -# 2. golint (https://github.com/golang/lint) -# 3. go vet (http://golang.org/cmd/vet) -# 4. gosimple (https://github.com/dominikh/go-simple) -# 5. unconvert (https://github.com/mdempsky/unconvert) -# 6. race detector (http://blog.golang.org/race-detector) -# 7. test coverage (http://blog.golang.org/cover) -# -# gometalinter (github.com/alecthomas/gometalinter) is used to run each static -# checker. - -declare GREEN='\033[0;32m' -declare NC='\033[0m' -print () { - echo -e "${GREEN}$1${NC}" -} - -# check_test_ports checks that test lnd ports are not used. -check_test_ports() { - # Make sure that test lnd ports are not used. - o=$(lsof -i :19555,19556 | sed '1d' | awk '{ printf "%s\n", $2 }') - if [ "$o" != "" ]; then - printf "Can't run the lnd tests:\n" - printf "some program is using the test lnd ports (19555 | 19556)\n" - exit 1 - fi -} - -# test_with_profile run test coverage on each subdirectories and merge the -# coverage profile. Be aware that we are skipping the integration tests, as the -# tool won't gather any useful coverage information from them. -test_with_coverage_profile() { - print "* Running tests with creating coverage profile" - - echo "mode: count" > profile.cov - - # Standard go tooling behavior is to ignore dirs with leading underscores. - for dir in $(find . -maxdepth 10 \ - -not -path './.git*' \ - -not -path '*/_*' \ - -not -path '.' \ - -not -path './cmd*' \ - -not -path './release*' \ - -not -path './vendor*' \ - -type d) - do - if ls $dir/*.go &> /dev/null; then - go test -v -covermode=count -coverprofile=$dir/profile.tmp $dir - - if [ -f $dir/profile.tmp ]; then - cat $dir/profile.tmp | tail -n +2 >> profile.cov - rm $dir/profile.tmp - fi - fi - done - - if [ "$TRAVIS" == "true" ] ; then - print "* Send test coverage to travis server" - # Make sure goveralls is installed and $GOPATH/bin is in your path. - if [ ! -x "$(type -p goveralls)" ]; then - print "** Install goveralls:" - go get github.com/mattn/goveralls - fi - - print "** Submit the test coverage result to coveralls.io" - goveralls -coverprofile=profile.cov -service=travis-ci - fi -} - -# test_race_conditions run standard go test without creating coverage -# profile but with race condition checks. -test_race_conditions() { - print "* Running tests with the race condition detector" - - test_targets=$(go list ./... | grep -v '/vendor/') - env GORACE="history_size=7 halt_on_error=1" go test -v -p 1 -race $test_targets -} - -# lint_check runs static checks. -lint_check() { - print "* Run static checks" - - # Make sure gometalinter is installed and $GOPATH/bin is in your path. - if [ ! -x "$(type -p gometalinter.v1)" ]; then - print "** Install gometalinter" - go get -u gopkg.in/alecthomas/gometalinter.v1 - gometalinter.v1 --install - fi - - # Update metalinter if needed. - gometalinter.v1 --install 1>/dev/null - - # Automatic checks - linter_targets=$(go list -f '{{.Dir}}' ./... | grep -v lnrpc) - test -z "$(gometalinter.v1 --disable-all \ - --enable=gofmt \ - --enable=vet \ - --enable=golint \ - --line-length=72 \ - --deadline=4m $linter_targets 2>&1 | grep -v 'ALL_CAPS\|OP_' 2>&1 | tee /dev/stderr)" -} - -set -e - -print "====+ Start +====" - -# Read input flags and initialize variables -NEED_LINT="false" -NEED_COVERAGE="false" -NEED_RACE="false" -NEED_INSTALL="false" -NEED_LOGS="false" - -while getopts "lrcio" flag; do - case "${flag}" in - l) NEED_LINT="true" ;; - r) NEED_RACE="true" ;; - c) NEED_COVERAGE="true" ;; - i) NEED_INSTALL="true" ;; - o) NEED_LOGS="true" ;; - *) - printf '\nUsage: %s [-l] [-r] [-c] [-i] [-o], where:\n' $0 - printf ' -l: include code lint check\n' - printf ' -r: run tests with race condition check\n' - printf ' -c: run tests with test coverage\n' - printf ' -i: reinstall project dependencies\n' - printf ' -o: generate logs for spawned lnd instances\n' - exit 1 ;; - esac - done - -# remove the options from the positional parameters -shift $(( OPTIND - 1 )) - -# Make sure dep is installed and $GOPATH/bin is in your path. -if [ ! -x "$(type -p dep)" ]; then - print "* Install dep" - go get -u github.com/golang/dep/cmd/dep -fi - -# Install the dependency if vendor directory not exist or if flag have been -# specified. -if [ "$NEED_INSTALL" == "true" ] || [ ! -d "./vendor" ]; then - print "* Install dependencies" - dep ensure -fi - -# Required for the integration tests -print "* Build source" -go install -v . ./cmd/... - -# Lint check is first because we shouldn't run tests on garbage code. -if [ "$NEED_LINT" == "true" ] || [ "$RACE" == "false" ]; then - lint_check -fi - -# Race condition second because we shouldn't check coverage on buggy code. -if [ "$NEED_RACE" == "true" ] || [ "$RACE" == "true" ]; then - test_race_conditions -fi - -# Test coverage is third because in this case code should work properly and -# we may calmly send coverage profile (if script is run on travis) -if [ "$NEED_COVERAGE" == "true" ] || [ "$RACE" == "false" ]; then - print "* Running integration tests" - - LOGOUTPUT_FLAG="" - if [ "$NEED_LOGS" == "true" ] || [ "$LOGS" == "true" ]; then - LOGOUTPUT_FLAG="-logoutput=true" - fi - - go test -v -tags rpctest "$LOGOUTPUT_FLAG" - - test_with_coverage_profile -fi - -print "=====+ End +=====" diff --git a/lntest/node.go b/lntest/node.go index 0b377b634..1122e0889 100644 --- a/lntest/node.go +++ b/lntest/node.go @@ -245,7 +245,7 @@ func (hn *HarnessNode) start(lndError chan<- error) error { args := hn.cfg.genArgs() args = append(args, fmt.Sprintf("--profile=%d", 9000+hn.NodeID)) - hn.cmd = exec.Command("lnd", args...) + hn.cmd = exec.Command("./lnd", args...) // Redirect stderr output to buffer var errb bytes.Buffer diff --git a/make/testing_flags.mk b/make/testing_flags.mk new file mode 100644 index 000000000..3c3f926b1 --- /dev/null +++ b/make/testing_flags.mk @@ -0,0 +1,45 @@ +TEST_FLAGS = + +# If specific package is being unit tested, construct the full name of the +# subpackage. +ifneq ($(pkg),) +UNITPKG := $(PKG)/$(pkg) +UNIT_TARGETED = yes +endif + +# If a specific unit test case is being target, construct test.run filter. +ifneq ($(case),) +TEST_FLAGS += -test.run=$(case) +UNIT_TARGETED = yes +endif + +# Define the integration test.run filter if the icase argument was provided. +ifneq ($(icase),) +TEST_FLAGS += -test.run=TestLightningNetworkDaemon/$(icase) +endif + +# If a timeout was requested, construct initialize the proper flag for the go +# test command. +ifneq ($(timeout),) +TEST_FLAGS += -test.timeout=$(timeout) +endif + +# UNIT_TARGTED is undefined iff a specific package and/or unit test case is +# not being targeted. +UNIT_TARGETED ?= no + +# If a specific package/test case was requested, run the unit test for the +# targeted case. Otherwise, default to running all tests. +ifeq ($(UNIT_TARGETED), yes) +UNIT := $(GOTEST) $(TEST_FLAGS) $(UNITPKG) +UNIT_RACE := $(GOTEST) $(TEST_FLAGS) -race $(UNITPKG) +endif + +ifeq ($(UNIT_TARGETED), no) +UNIT := $(GOLIST) | $(XARGS) $(GOTEST) $(TEST_FLAGS) +UNIT_RACE := $(UNIT) -race +endif + + +# Construct the integration test command with the added build flags. +ITEST := $(GOTEST) $(TEST_FLAGS) -tags rpctest -logoutput