mirrors/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-20 14:05:23 +01:00
Code Issues Projects Releases Wiki Activity

Merge #19205: script: previous_release.sh rewritten in python

Browse source 
9c34aff393 Remove previous_release.sh (Brian Liotti)
e1e5960e10 script: Add previous_release.py (Brian Liotti)

Pull request description:

  Closes #18132

  Added functionality:
  1) checks file hash before untarring when using the binary download option

ACKs for top commit:
  fjahr:
    re-ACK 9c34aff393
  Sjors:
    tACK 9c34aff393

Tree-SHA512: 323f11828736a372a47f048592de8b027ddcd75b38f312dfc73f7b495d1e078bfeb384d9cdf434b3e70f2c6c0ce2da2df48e9a6460ac0e1967c6829a411c52d5
This commit is contained in:
MarcoFalke 2020-07-21 09:48:12 +02:00
commit ea595d39f7
No known key found for this signature in database
GPG key ID: CE2B75697E69A548
6 changed files with 226 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ci/test/05_before_script.sh
View file

@ -48,6 +48,6 @@ if [ -z "$NO_DEPENDS" ]; then
fi
if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
BEGIN_FOLD previous-versions
DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
END_FOLD
fi

222
contrib/devtools/previous_release.py Executable file
View file

@ -0,0 +1,222 @@
#!/usr/bin/env python3
#
# Copyright (c) 2018-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Build previous releases.
import argparse
import contextlib
from fnmatch import fnmatch
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import hashlib
@contextlib.contextmanager
def pushd(new_dir) -> None:
previous_dir = os.getcwd()
os.chdir(new_dir)
try:
yield
finally:
os.chdir(previous_dir)
def download_binary(tag, args) -> int:
if Path(tag).is_dir():
if not args.remove_dir:
print('Using cached {}'.format(tag))
return 0
shutil.rmtree(tag)
Path(tag).mkdir()
bin_path = 'bin/bitcoin-core-{}'.format(tag[1:])
match = re.compile('v(.*)(rc[0-9]+)$').search(tag)
if match:
bin_path = 'bin/bitcoin-core-{}/test.{}'.format(
match.group(1), match.group(2))
tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format(
tag=tag[1:], platform=args.platform)
sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:])
tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format(
bin_path=bin_path, tarball=tarball)
sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format(
bin_path=bin_path)
print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl))
print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl))
header, status = subprocess.Popen(
['curl', '-I', tarballUrl], stdout=subprocess.PIPE).communicate()
if re.search("404 Not Found", header.decode("utf-8")):
print("Binary tag was not found")
return 1
curlCmds = [
['curl', '-O', tarballUrl],
['curl', "-o", sha256Sums, sha256SumsUrl],
]
for cmd in curlCmds:
ret = subprocess.run(cmd).returncode
if ret:
return ret
hasher = hashlib.sha256()
with open(tarball, "rb") as afile:
buf = afile.read()
hasher.update(buf)
afile.close()
tarballHash = hasher.hexdigest()
file = open(sha256Sums, 'r', encoding="utf-8")
lst = list(file.readlines())
file.close()
lastline = lst[len(lst)-1]
for line in lst:
if re.search(tarballHash, line):
print("Checksum matched")
break
elif lastline == line:
print("Checksum did not match")
Path(tarball).unlink()
return 1
# Bitcoin Core Release Signing Keys v0.11.0+
signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964"
isKeyPresent = subprocess.run(
["gpg", "--list-keys", signingKey]).returncode
if isKeyPresent:
return isKeyPresent
isVerified = subprocess.run(
["gpg", "--verify", sha256Sums]).returncode
if isVerified:
return isVerified
# Extract tarball
ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag,
'--strip-components=1',
'bitcoin-{tag}'.format(tag=tag[1:])]).returncode
if ret:
return ret
Path(tarball).unlink()
Path(sha256Sums).unlink()
return 0
def build_release(tag, args) -> int:
githubUrl = "https://github.com/bitcoin/bitcoin"
if args.remove_dir:
if Path(tag).is_dir():
shutil.rmtree(tag)
if not Path(tag).is_dir():
# fetch new tags
subprocess.run(
["git", "fetch", githubUrl, "--tags"])
output = subprocess.check_output(['git', 'tag', '-l', tag])
if not output:
print('Tag {} not found'.format(tag))
return 1
ret = subprocess.run([
'git', 'clone', githubUrl, tag
]).returncode
if ret:
return ret
with pushd(tag):
ret = subprocess.run(['git', 'checkout', tag]).returncode
if ret:
return ret
host = args.host
if args.depends:
with pushd('depends'):
ret = subprocess.run(['make', 'NO_QT=1']).returncode
if ret:
return ret
host = os.environ.get(
'HOST', subprocess.check_output(['./config.guess']))
config_flags = '--prefix={pwd}/depends/{host} '.format(
pwd=os.getcwd(),
host=host) + args.config_flags
cmds = [
'./autogen.sh',
'./configure {}'.format(config_flags),
'make',
]
for cmd in cmds:
ret = subprocess.run(cmd.split()).returncode
if ret:
return ret
# Move binaries, so they're in the same place as in the
# release download
Path('bin').mkdir(exist_ok=True)
files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx']
for f in files:
Path('src/'+f).rename('bin/'+f)
return 0
def check_host(args) -> int:
args.host = os.environ.get('HOST', subprocess.check_output(
'./depends/config.guess').decode())
if args.download_binary:
platforms = {
'x86_64-*-linux*': 'x86_64-linux-gnu',
'x86_64-apple-darwin*': 'osx64',
}
args.platform = ''
for pattern, target in platforms.items():
if fnmatch(args.host, pattern):
args.platform = target
if not args.platform:
print('Not sure which binary to download for {}'.format(args.host))
return 1
return 0
def main(args) -> int:
if not Path(args.target_dir).is_dir():
Path(args.target_dir).mkdir(exist_ok=True, parents=True)
print("Releases directory: {}".format(args.target_dir))
ret = check_host(args)
if ret:
return ret
if args.download_binary:
with pushd(args.target_dir):
for tag in args.tags:
ret = download_binary(tag, args)
if ret:
return ret
return 0
args.config_flags = os.environ.get('CONFIG_FLAGS', '')
args.config_flags += ' --without-gui --disable-tests --disable-bench'
with pushd(args.target_dir):
for tag in args.tags:
ret = build_release(tag, args)
if ret:
return ret
return 0
if __name__ == '__main__':
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-r', '--remove-dir', action='store_true',
help='remove existing directory.')
parser.add_argument('-d', '--depends', action='store_true',
help='use depends.')
parser.add_argument('-b', '--download-binary', action='store_true',
help='download release binary.')
parser.add_argument('-t', '--target-dir', action='store',
help='target directory.', default='releases')
parser.add_argument('tags', nargs='+',
help="release tags. e.g.: v0.18.1 v0.20.0rc2")
args = parser.parse_args()
sys.exit(main(args))

152
contrib/devtools/previous_release.sh
View file

@ -1,152 +0,0 @@
#!/usr/bin/env bash
#
# Copyright (c) 2018-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Build previous releases.
export LC_ALL=C
CONFIG_FLAGS=""
FUNCTIONAL_TESTS=0
DELETE_EXISTING=0
USE_DEPENDS=0
DOWNLOAD_BINARY=0
CONFIG_FLAGS=""
TARGET="releases"
while getopts ":hfrdbt:" opt; do
case $opt in
h)
echo "Usage: .previous_release.sh [options] tag1 tag2"
echo " options:"
echo " -h Print this message"
echo " -f Configure for functional tests"
echo " -r Remove existing directory"
echo " -d Use depends"
echo " -b Download release binary"
echo " -t Target directory (default: releases)"
exit 0
;;
f)
FUNCTIONAL_TESTS=1
CONFIG_FLAGS="$CONFIG_FLAGS --without-gui --disable-tests --disable-bench"
;;
r)
DELETE_EXISTING=1
;;
d)
USE_DEPENDS=1
;;
b)
DOWNLOAD_BINARY=1
;;
t)
TARGET=$OPTARG
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
esac
done
shift $((OPTIND-1))
if [ -z "$1" ]; then
echo "Specify release tag(s), e.g.: .previous_release v0.15.1"
exit 1
fi
if [ ! -d "$TARGET" ]; then
mkdir -p $TARGET
fi
if [ "$DOWNLOAD_BINARY" -eq "1" ]; then
HOST="${HOST:-$(./depends/config.guess)}"
case "$HOST" in
x86_64-*-linux*)
PLATFORM=x86_64-linux-gnu
;;
x86_64-apple-darwin*)
PLATFORM=osx64
;;
*)
echo "Not sure which binary to download for $HOST."
exit 1
;;
esac
fi
echo "Releases directory: $TARGET"
pushd "$TARGET" || exit 1
{
for tag in "$@"
do
if [ "$DELETE_EXISTING" -eq "1" ]; then
if [ -d "$tag" ]; then
rm -r "$tag"
fi
fi
if [ "$DOWNLOAD_BINARY" -eq "0" ]; then
if [ ! -d "$tag" ]; then
if [ -z $(git tag -l "$tag") ]; then
echo "Tag $tag not found"
exit 1
fi
git clone https://github.com/bitcoin/bitcoin "$tag"
pushd "$tag" || exit 1
{
git checkout "$tag"
if [ "$USE_DEPENDS" -eq "1" ]; then
pushd depends || exit 1
{
if [ "$FUNCTIONAL_TESTS" -eq "1" ]; then
make NO_QT=1
else
make
fi
HOST="${HOST:-$(./config.guess)}"
}
popd || exit 1
CONFIG_FLAGS="--prefix=$PWD/depends/$HOST $CONFIG_FLAGS"
fi
./autogen.sh
./configure $CONFIG_FLAGS
make
# Move binaries, so they're in the same place as in the release download:
mkdir bin
mv src/bitcoind src/bitcoin-cli src/bitcoin-tx bin
if [ "$FUNCTIONAL_TESTS" -eq "0" ]; then
mv src/qt/bitcoin-qt bin
fi
}
popd || exit 1
fi
else
if [ -d "$tag" ]; then
echo "Using cached $tag"
else
mkdir "$tag"
if [[ "$tag" =~ v(.*)(rc[0-9]+)$ ]]; then
BIN_PATH="bin/bitcoin-core-${BASH_REMATCH[1]}/test.${BASH_REMATCH[2]}"
else
BIN_PATH="bin/bitcoin-core-${tag:1}"
fi
URL="https://bitcoincore.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz"
echo "Fetching: $URL"
if ! curl -O -f $URL; then
echo "Download failed."
exit 1
fi
tar -zxf "bitcoin-${tag:1}-$PLATFORM.tar.gz" -C "$tag" --strip-components=1 "bitcoin-${tag:1}"
rm "bitcoin-${tag:1}-$PLATFORM.tar.gz"
fi
fi
done
}
popd || exit 1

2
test/functional/feature_backwards_compatibility.py
View file

@ -6,7 +6,7 @@
Test various backwards compatibility scenarios. Download the previous node binaries:
contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py.
Due to a hardfork in regtest, it can't be used to sync nodes.

2
test/functional/mempool_compatibility.py
View file

@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden
In case we need to break mempool compatibility we can continue to use the test by just bumping the version number.
Download node binaries:
contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests.
"""

2
test/functional/wallet_upgradewallet.py
View file

@ -6,7 +6,7 @@
Test upgradewallet RPC. Download node binaries:
contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py
"""