Merge #17361: script: Lint Gitian descriptors with ShellCheck

17f81e9648 script: Enable SC2001 rule for Gitian scripts (Hennadii Stepanov)
61bb21b418 script: Enable SC2155 rule for Gitian scripts (Hennadii Stepanov)
577682d9e8 script: Enable SC2006 rule for Gitian scripts (Hennadii Stepanov)
14aded46df script: Lint Gitian descriptors with ShellCheck (Hennadii Stepanov)

Pull request description:

  This PR extracts shell scripts from Gitian descriptors (`contrib/gitian-descriptors/`) and checks for ShellCheck warnings as any other one.

  Some non-controversial warnings are fixed.

ACKs for top commit:
  practicalswift:
    ACK 17f81e9648 -- diff looks correct

Tree-SHA512: bdfa3d35bbb65ff634c90835d75c3df63e958b558599771d21366724f5cf64da83a68957d926e926a99c3704b9529e96a17697dc8d9ff3adf7154d9cb1999a8d

ci/lint/04_install.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -8,6 +8,7 @@ export LC_ALL=C
 
 travis_retry pip3 install codespell==1.15.0
 travis_retry pip3 install flake8==3.7.8
+travis_retry pip3 install yq
 
 SHELLCHECK_VERSION=v0.6.0
 curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/

contrib/gitian-descriptors/gitian-linux.yml

@@ -51,7 +51,7 @@ script: |
   export QT_RCC_TEST=1
   export QT_RCC_SOURCE_DATE_OVERRIDE=1
   export TZ="UTC"
-  export BUILD_DIR=`pwd`
+  export BUILD_DIR="$PWD"
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
@@ -108,7 +108,7 @@ script: |
   rm -f ${WRAP_DIR}/${prog}
   cat << EOF > ${WRAP_DIR}/${prog}
   #!/usr/bin/env bash
-  REAL="`which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1`"
+  REAL="$(which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1)"
   for var in "\$@"
   do
     if [ "\$var" = "-m32" ]; then
@@ -123,7 +123,7 @@ script: |
   done
 
   cd bitcoin
-  BASEPREFIX=`pwd`/depends
+  BASEPREFIX="${PWD}/depends"
   # Build dependencies for each host
   for i in $HOSTS; do
     EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
@@ -142,10 +142,11 @@ script: |
 
   # Create the release tarball using (arbitrarily) the first host
   ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/
+  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
   make dist
-  SOURCEDIST=`echo bitcoin-*.tar.gz`
+  SOURCEDIST=$(echo bitcoin-*.tar.gz)
-  DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
+  DISTNAME=${SOURCEDIST/%.tar.gz}
+
   # Correct tar file order
   mkdir -p temp
   pushd temp
@@ -169,7 +170,7 @@ script: |
     fi
     mkdir -p distsrc-${i}
     cd distsrc-${i}
-    INSTALLPATH=`pwd`/installed/${DISTNAME}
+    INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
     tar --strip-components=1 -xf ../$SOURCEDIST
 

contrib/gitian-descriptors/gitian-osx-signer.yml

@@ -17,7 +17,7 @@ script: |
 
   WRAP_DIR=$HOME/wrapped
   mkdir -p ${WRAP_DIR}
-  export PATH=`pwd`:$PATH
+  export PATH="$PWD":$PATH
   FAKETIME_PROGS="dmg genisoimage"
 
   # Create global faketime wrappers

contrib/gitian-descriptors/gitian-osx.yml

@@ -45,7 +45,7 @@ script: |
   export QT_RCC_TEST=1
   export QT_RCC_SOURCE_DATE_OVERRIDE=1
   export TZ="UTC"
-  export BUILD_DIR=`pwd`
+  export BUILD_DIR="$PWD"
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
@@ -87,7 +87,7 @@ script: |
   export PATH=${WRAP_DIR}:${PATH}
 
   cd bitcoin
-  BASEPREFIX=`pwd`/depends
+  BASEPREFIX="${PWD}/depends"
 
   mkdir -p ${BASEPREFIX}/SDKs
   tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz
@@ -105,10 +105,10 @@ script: |
 
   # Create the release tarball using (arbitrarily) the first host
   ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/
+  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
   make dist
-  SOURCEDIST=`echo bitcoin-*.tar.gz`
+  SOURCEDIST=$(echo bitcoin-*.tar.gz)
-  DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
+  DISTNAME=${SOURCEDIST/%.tar.gz}
 
   # Correct tar file order
   mkdir -p temp
@@ -126,7 +126,7 @@ script: |
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir -p distsrc-${i}
     cd distsrc-${i}
-    INSTALLPATH=`pwd`/installed/${DISTNAME}
+    INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
     tar --strip-components=1 -xf ../$SOURCEDIST
 

contrib/gitian-descriptors/gitian-win-signer.yml

@@ -19,7 +19,7 @@ files:
 script: |
   set -e -o pipefail
 
-  BUILD_DIR=`pwd`
+  BUILD_DIR="$PWD"
   SIGDIR=${BUILD_DIR}/signature/win
   UNSIGNED_DIR=${BUILD_DIR}/unsigned
 
@@ -35,7 +35,7 @@ script: |
   ./configure --without-gsf --without-curl --disable-dependency-tracking
   make
   find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do
-    INFILE="`basename "${i}"`"
+    INFILE="$(basename "${i}")"
-    OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`"
+    OUTFILE="${INFILE/%-unsigned}"
     ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem"
   done

contrib/gitian-descriptors/gitian-win.yml

@@ -40,7 +40,7 @@ script: |
   export QT_RCC_TEST=1
   export QT_RCC_SOURCE_DATE_OVERRIDE=1
   export TZ="UTC"
-  export BUILD_DIR=`pwd`
+  export BUILD_DIR="$PWD"
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
@@ -97,7 +97,7 @@ script: |
   export PATH=${WRAP_DIR}:${PATH}
 
   cd bitcoin
-  BASEPREFIX=`pwd`/depends
+  BASEPREFIX="${PWD}/depends"
   # Build dependencies for each host
   for i in $HOSTS; do
     make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
@@ -112,10 +112,11 @@ script: |
 
   # Create the release tarball using (arbitrarily) the first host
   ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/
+  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
   make dist
-  SOURCEDIST=`echo bitcoin-*.tar.gz`
+  SOURCEDIST=$(echo bitcoin-*.tar.gz)
-  DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
+  DISTNAME=${SOURCEDIST/%.tar.gz}
+
   # Correct tar file order
   mkdir -p temp
   pushd temp
@@ -132,7 +133,7 @@ script: |
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir -p distsrc-${i}
     cd distsrc-${i}
-    INSTALLPATH=`pwd`/installed/${DISTNAME}
+    INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
     tar --strip-components=1 -xf ../$SOURCEDIST
 

test/README.md

@@ -258,6 +258,7 @@ Use the `-v` option for verbose output.
 |-----------|:----------:|:-------------------------------------------:|--------------
 | [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.7.8](https://github.com/bitcoin/bitcoin/pull/15257) | `pip3 install flake8==3.7.8`
 | [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.6.0](https://github.com/bitcoin/bitcoin/pull/15166) | [details...](https://github.com/koalaman/shellcheck#installing)
+| [`lint-shell.sh`](lint/lint-shell.sh) | [yq](https://github.com/kislyuk/yq) | default | `pip3 install yq`
 | [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.15.0](https://github.com/bitcoin/bitcoin/pull/16186) | `pip3 install codespell==1.15.0`
 
 Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated.

test/lint/lint-shell.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
@@ -16,16 +16,45 @@ if [ "$TRAVIS" = "true" ]; then
     unset LC_ALL
 fi
 
-if ! command -v shellcheck > /dev/null; then
-    echo "Skipping shell linting since shellcheck is not installed."
-    exit 0
-fi
-
 # Disabled warnings:
 disabled=(
     SC2046 # Quote this to prevent word splitting.
     SC2086 # Double quote to prevent globbing and word splitting.
     SC2162 # read without -r will mangle backslashes.
 )
-shellcheck -e "$(IFS=","; echo "${disabled[*]}")" \
+disabled_gitian=(
-    $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/')
+    SC2094 # Make sure not to read and write the same file in the same pipeline.
+    SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects.
+    SC2230 # which is non-standard. Use builtin 'command -v' instead.
+)
+
+EXIT_CODE=0
+
+if ! command -v shellcheck > /dev/null; then
+    echo "Skipping shell linting since shellcheck is not installed."
+    exit $EXIT_CODE
+fi
+
+EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")"
+if ! shellcheck "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then
+    EXIT_CODE=1
+fi
+
+if ! command -v yq > /dev/null; then
+    echo "Skipping Gitian desriptor scripts checking since yq is not installed."
+    exit $EXIT_CODE
+fi
+
+EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")"
+for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml')
+do
+    echo
+    echo "$descriptor"
+    # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script.
+    SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor")
+    if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then
+        EXIT_CODE=1
+    fi
+done
+
+exit $EXIT_CODE