From 37c9fb7a59a3179b90ed1deaebaabb539976504b Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 16 Sep 2021 18:33:20 -0400 Subject: [PATCH 01/14] contrib: verifybinaries: allow multisig verification This commit adds the functionality necessary to transition from doing binary verification on the basis of a single signature to requiring a minimum threshold of trusted signatures. A signature can appear as "good" from GPG output, but it may not come from an identity the user trusts. We call these "good, untrusted" signatures. We report bad signatures but do not necessarily fail in their presence, since a bad signature might coexist with enough good, trusted signatures to fulfill our criteria. If "--import-keys" is enabled, we will prompt the user to optionally try to retrieve unknown keys. Marking them as trusted locally is a WIP, but keys which are retrieved successfully and appear on the builder-keys list will immediately count as being useful towards fulfilling the threshold. Logging is improved and an option to output JSON that summarizes the whole sum signature and binary verification processes has been added. Co-authored-by: Russ Yanofsky Co-authored-by: willcl-ark --- contrib/verifybinaries/README.md | 69 +++- contrib/verifybinaries/test.py | 56 +++ contrib/verifybinaries/verify.py | 624 ++++++++++++++++++++++++++----- 3 files changed, 641 insertions(+), 108 deletions(-) create mode 100755 contrib/verifybinaries/test.py diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index ab831eea280..1579e69fc72 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -1,30 +1,77 @@ ### Verify Binaries -#### Usage: +#### Preparation -This script attempts to download the signature file `SHA256SUMS.asc` from https://bitcoin.org. +As of Bitcoin Core v22.0, releases are signed by a number of public keys on the basis +of the [guix.sigs repository](https://github.com/bitcoin-core/guix.sigs/). When +verifying binary downloads, you (the end user) decide which of these public keys you +trust and then use that trust model to evaluate the signature on a file that contains +hashes of the release binaries. The downloaded binaries are then hashed and compared to +the signed checksum file. -It first checks if the signature passes, and then downloads the files specified in the file, and checks if the hashes of these files match those that are specified in the signature file. +First, you have to figure out which public keys to recognize. Browse the [list of frequent +builder-keys](https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys) and +decide which of these keys you would like to trust. For each key you want to trust, you +must obtain that key for your local GPG installation. -The script returns 0 if everything passes the checks. It returns 1 if either the signature check or the hash check doesn't pass. If an error occurs the return value is 2. +You can obtain these keys by + - through a browser using a key server (e.g. keyserver.ubuntu.com), + - manually using the `gpg --keyserver --recv-keys ` command, or + - you can run the packaged `verifybinaries.py ... --import-keys` script to + have it automatically retrieve unrecognized keys. +#### Usage + +This script attempts to download the checksum file (`SHA256SUMS`) and corresponding +signature file `SHA256SUMS.asc` from https://bitcoincore.org and https://bitcoin.org. + +It first checks if the checksum file is valid based upon a plurality of signatures, and +then downloads the release files specified in the checksum file, and checks if the +hashes of the release files are as expected. + +If we encounter pubkeys in the signature file that we do not recognize, the script +can prompt the user as to whether they'd like to download the pubkeys. To enable +this behavior, use the `--import-keys` flag. + +The script returns 0 if everything passes the checks. It returns 1 if either the +signature check or the hash check doesn't pass. An exit code of >2 indicates an error. + +See the `Config` object for various options. + +#### Examples + +Validate releases with default settings: +```sh +./contrib/verifybinaries/verify.py 22.0 +./contrib/verifybinaries/verify.py 22.0-rc2 +./contrib/verifybinaries/verify.py bitcoin-core-23.0 +./contrib/verifybinaries/verify.py bitcoin-core-23.0-rc1 +``` + +Get JSON output and don't prompt for user input (no auto key import): ```sh -./verify.py bitcoin-core-0.11.2 -./verify.py bitcoin-core-0.12.0 -./verify.py bitcoin-core-0.13.0-rc3 +./contrib/verifybinaries/verify.py 22.0-x86 --json +``` + +Don't trust builder-keys by default, and rely only on local GPG state and manually +specified keys, while requiring a threshold of at least 10 trusted signatures: +```sh +./contrib/verifybinaries/verify.py 22.0-x86 \ + --no-trust-builder-keys \ + --trusted-keys 74E2DEF5D77260B98BC19438099BAD163C70FBFA,9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ + --min-trusted-sigs 10 ``` If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: ```sh -./verify.py bitcoin-core-0.11.2-osx -./verify.py 0.12.0-linux -./verify.py bitcoin-core-0.13.0-rc3-win64 +./contrib/verifybinaries/verify.py bitcoin-core-22.0-osx +./contrib/verifybinaries/verify.py bitcoin-core-22.0-rc2-win64 ``` If you do not want to keep the downloaded binaries, specify anything as the second parameter. ```sh -./verify.py bitcoin-core-0.13.0 delete +./contrib/verifybinaries/verify.py bitcoin-core-22.0 delete ``` diff --git a/contrib/verifybinaries/test.py b/contrib/verifybinaries/test.py new file mode 100755 index 00000000000..539dff658a7 --- /dev/null +++ b/contrib/verifybinaries/test.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import json +import sys +import subprocess +from pathlib import Path + + +def main(): + """Tests ordered roughly from faster to slower.""" + expect_code(run_verify('0.32'), 4, "Nonexistent version should fail") + expect_code(run_verify('0.32.awefa.12f9h'), 11, "Malformed version should fail") + expect_code(run_verify('22.0 --min-good-sigs 20'), 9, "--min-good-sigs 20 should fail") + + print("- testing multisig verification (22.0)", flush=True) + _220 = run_verify('22.0 --json') + try: + result = json.loads(_220.stdout.decode()) + except Exception: + print("failed on 22.0 --json:") + print_process_failure(_220) + raise + + expect_code(_220, 0, "22.0 should succeed") + v = result['verified_binaries'] + assert result['good_trusted_sigs'] + assert v['bitcoin-22.0-aarch64-linux-gnu.tar.gz'] == 'ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e' + assert v['bitcoin-22.0-osx64.tar.gz'] == '2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9' + assert v['bitcoin-22.0-x86_64-linux-gnu.tar.gz'] == '59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16' + + +def run_verify(extra: str) -> subprocess.CompletedProcess: + maybe_here = Path.cwd() / 'verify.py' + path = maybe_here if maybe_here.exists() else Path.cwd() / 'contrib' / 'verifybinaries' / 'verify.py' + + return subprocess.run( + f"{path} --cleanup {extra}", + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + + +def expect_code(completed: subprocess.CompletedProcess, expected_code: int, msg: str): + if completed.returncode != expected_code: + print(f"{msg!r} failed: got code {completed.returncode}, expected {expected_code}") + print_process_failure(completed) + sys.exit(1) + else: + print(f"✓ {msg!r} passed") + + +def print_process_failure(completed: subprocess.CompletedProcess): + print(f"stdout:\n{completed.stdout.decode()}") + print(f"stderr:\n{completed.stderr.decode()}") + + +if __name__ == '__main__': + main() diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index b5e4f1318b9..6b5fee43ee7 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -2,29 +2,154 @@ # Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Script for verifying Bitcoin Core release binaries +"""Script for verifying Bitcoin Core release binaries. -This script attempts to download the signature file SHA256SUMS.asc from -bitcoincore.org and bitcoin.org and compares them. -It first checks if the signature passes, and then downloads the files -specified in the file, and checks if the hashes of these files match those -that are specified in the signature file. -The script returns 0 if everything passes the checks. It returns 1 if either -the signature check or the hash check doesn't pass. If an error occurs the -return value is >= 2. +This script attempts to download the sum file SHA256SUMS and corresponding +signature file SHA256SUMS.asc from bitcoincore.org and bitcoin.org and +compares them. + +The sum-signature file is signed by a number of builder keys. This script +ensures that there is a minimum threshold of signatures from pubkeys that +we trust. This trust is articulated on the basis of configuration options +here, but by default is based upon local GPG trust settings. + +The builder keys are available in the guix.sigs repo: + + https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys + +If a minimum good, trusted signature threshold is met on the sum file, we then +download the files specified in SHA256SUMS, and check if the hashes of these +files match those that are specified. The script returns 0 if everything passes +the checks. It returns 1 if either the signature check or the hash check +doesn't pass. If an error occurs the return value is >= 2. + +Logging output goes to stderr and final binary verification data goes to stdout. + +JSON output can by obtained by setting env BINVERIFY_JSON=1. """ -from hashlib import sha256 +import argparse +import difflib +import json +import logging import os import subprocess +import typing as t +import re import sys -from textwrap import indent +import shutil +import tempfile +import textwrap +import urllib.request +import enum +from hashlib import sha256 +from pathlib import Path -WORKINGDIR = "/tmp/bitcoin_verify_binaries" -HASHFILE = "hashes.tmp" +# The primary host; this will fail if we can't retrieve files from here. HOST1 = "https://bitcoincore.org" HOST2 = "https://bitcoin.org" VERSIONPREFIX = "bitcoin-core-" -SIGNATUREFILENAME = "SHA256SUMS.asc" +SUMS_FILENAME = 'SHA256SUMS' +SIGNATUREFILENAME = f"{SUMS_FILENAME}.asc" + + +class ReturnCode(enum.IntEnum): + SUCCESS = 0 + INTEGRITY_FAILURE = 1 + FILE_GET_FAILED = 4 + FILE_MISSING_FROM_ONE_HOST = 5 + FILES_NOT_EQUAL = 6 + NO_BINARIES_MATCH = 7 + NOT_ENOUGH_GOOD_SIGS = 9 + BINARY_DOWNLOAD_FAILED = 10 + BAD_VERSION = 11 + + +def set_up_logger(is_verbose: bool = True) -> logging.Logger: + """Set up a logger that writes to stderr.""" + log = logging.getLogger(__name__) + log.setLevel(logging.INFO if is_verbose else logging.WARNING) + console = logging.StreamHandler(sys.stderr) # log to stderr + console.setLevel(logging.DEBUG) + formatter = logging.Formatter('[%(levelname)s] %(message)s') + console.setFormatter(formatter) + log.addHandler(console) + return log + + +log = set_up_logger() + + +def indent(output: str) -> str: + return textwrap.indent(output, ' ') + + +def bool_from_env(key, default=False) -> bool: + if key not in os.environ: + return default + raw = os.environ[key] + + if raw.lower() in ('1', 'true'): + return True + elif raw.lower() in ('0', 'false'): + return False + raise ValueError(f"Unrecognized environment value {key}={raw!r}") + + +VERSION_FORMAT = ".[.][-rc[0-9]][-platform]" +VERSION_EXAMPLE = "22.0-x86_64 or 0.21.0-rc2-osx" + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + 'version', type=str, help=( + f'version of the bitcoin release to download; of the format ' + f'{VERSION_FORMAT}. Example: {VERSION_EXAMPLE}') +) +parser.add_argument( + '-v', '--verbose', action='store_true', + default=bool_from_env('BINVERIFY_VERBOSE'), +) +parser.add_argument( + '-q', '--quiet', action='store_true', + default=bool_from_env('BINVERIFY_QUIET'), +) +parser.add_argument( + '--cleanup', action='store_true', + default=bool_from_env('BINVERIFY_CLEANUP'), + help='if specified, clean up files afterwards' +) +parser.add_argument( + '--import-keys', action='store_true', + default=bool_from_env('BINVERIFY_IMPORTKEYS'), + help='if specified, ask to import each unknown builder key' +) +parser.add_argument( + '--require-all-hosts', action='store_true', + default=bool_from_env('BINVERIFY_REQUIRE_ALL_HOSTS'), + help=( + f'If set, require all hosts ({HOST1}, {HOST2}) to provide signatures. ' + '(Sometimes bitcoin.org lags behind bitcoincore.org.)') +) +parser.add_argument( + '--min-good-sigs', type=int, action='store', nargs='?', + default=int(os.environ.get('BINVERIFY_MIN_GOOD_SIGS', 3)), + help=( + 'The minimum number of good signatures to require successful termination.'), +) +parser.add_argument( + '--keyserver', action='store', nargs='?', + default=os.environ.get('BINVERIFY_KEYSERVER', 'hkp://keyserver.ubuntu.com'), + help='which keyserver to use', +) +parser.add_argument( + '--trusted-keys', action='store', nargs='?', + default=os.environ.get('BINVERIFY_TRUSTED_KEYS', ''), + help='A list of trusted signer GPG keys, separated by commas. Not "trusted keys" in the GPG sense.', +) +parser.add_argument( + '--json', action='store_true', + default=bool_from_env('BINVERIFY_JSON'), + help='If set, output the result as JSON', +) def parse_version_string(version_str): @@ -47,30 +172,35 @@ def parse_version_string(version_str): return version_base, version_rc, version_os -def download_with_wget(remote_file, local_file=None): - if local_file: - wget_args = ['wget', '-O', local_file, remote_file] - else: - # use timestamping mechanism if local filename is not explicitly set - wget_args = ['wget', '-N', remote_file] - - result = subprocess.run(wget_args, +def download_with_wget(remote_file, local_file): + result = subprocess.run(['wget', '-O', local_file, remote_file], stderr=subprocess.STDOUT, stdout=subprocess.PIPE) return result.returncode == 0, result.stdout.decode().rstrip() -def files_are_equal(filename1, filename2): - with open(filename1, 'rb') as file1: - contents1 = file1.read() - with open(filename2, 'rb') as file2: - contents2 = file2.read() - return contents1 == contents2 +def download_lines_with_urllib(url) -> t.Tuple[bool, t.List[str]]: + """Get (success, text lines of a file) over HTTP.""" + try: + return (True, [ + line.strip().decode() for line in urllib.request.urlopen(url).readlines()]) + except urllib.request.HTTPError as e: + log.warning(f"HTTP request to {url} failed (HTTPError): {e}") + except Exception as e: + log.warning(f"HTTP request to {url} failed ({e})") + return (False, []) -def verify_with_gpg(signature_filename, output_filename): - result = subprocess.run(['gpg', '--yes', '--decrypt', '--output', - output_filename, signature_filename], - stderr=subprocess.STDOUT, stdout=subprocess.PIPE) +def verify_with_gpg( + signature_filename, + output_filename: t.Optional[str] = None +) -> t.Tuple[int, str]: + args = [ + 'gpg', '--yes', '--decrypt', '--verify-options', 'show-primary-uid-only', + '--output', output_filename if output_filename else '', signature_filename] + + env = dict(os.environ, LANGUAGE='en') + result = subprocess.run(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env) + log.debug(f'Result from GPG ({result.returncode}): {result.stdout}') return result.returncode, result.stdout.decode().rstrip() @@ -79,104 +209,404 @@ def remove_files(filenames): os.remove(filename) +class SigData: + """GPG signature data as parsed from GPG stdout.""" + def __init__(self, key: str, name: str, trusted: bool, status: str): + self.key = key + self.name = name + self.trusted = trusted + self.status = status + + def __repr__(self): + return ( + "SigData(%r, %r, trusted=%s, status=%r)" % + (self.key, self.name, self.trusted, self.status)) + + +def parse_gpg_result( + output: t.List[str] +) -> t.Tuple[t.List[SigData], t.List[SigData], t.List[SigData]]: + """Returns good, unknown, and bad signatures from GPG stdout.""" + good_sigs = [] + unknown_sigs = [] + bad_sigs = [] + total_resolved_sigs = 0 + curr_key = None + + # Ensure that all lines we match on include a prefix that prevents malicious input + # from fooling the parser. + def line_begins_with(patt: str, line: str) -> t.Optional[re.Match]: + return re.match(r'^\s*(gpg:)?(\s+)' + patt, line) + + detected_name = '' + + for i, line in enumerate(output): + if line_begins_with(r"using (ECDSA|RSA) key (0x[0-9a-fA-F]{16}|[0-9a-fA-F]{40})$", line): + if curr_key: + raise RuntimeError( + f"WARNING: encountered a new sig without resolving the last ({curr_key}) - " + "this could mean we have encountered a bad signature! check GPG output!") + curr_key = line.split('key ')[-1].strip() + assert len(curr_key) == 40 or (len(curr_key) == 18 and curr_key.startswith('0x')) + + if line_begins_with(r"Can't check signature: No public key$", line): + if not curr_key: + raise RuntimeError("failed to detect signature being resolved") + unknown_sigs.append(SigData(curr_key, detected_name, False, '')) + detected_name = '' + curr_key = None + + if line_begins_with(r'Good signature from (".+")(\s+)(\[.+\])$', line): + if not curr_key: + raise RuntimeError("failed to detect signature being resolved") + name, status = parse_gpg_from_line(line) + + # It's safe to index output[i + 1] because if we saw a good sig, there should + # always be another line + trusted = ( + 'This key is not certified with a trusted signature' not in output[i + 1]) + good_sigs.append(SigData(curr_key, name, trusted, status)) + curr_key = None + + if line_begins_with("issuer ", line): + detected_name = line.split("issuer ")[-1].strip('"') + + if 'bad signature from' in line.lower(): + if not curr_key: + raise RuntimeError("failed to detect signature being resolved") + name, status = parse_gpg_from_line(line) + bad_sigs.append(SigData(curr_key, name, False, status)) + curr_key = None + + # Track total signatures included + if line_begins_with('Signature made ', line): + total_resolved_sigs += 1 + + all_found = len(good_sigs + bad_sigs + unknown_sigs) + if all_found != total_resolved_sigs: + raise RuntimeError( + f"failed to evaluate all signatures: found {all_found} " + f"but expected {total_resolved_sigs}") + + return (good_sigs, unknown_sigs, bad_sigs) + + +def parse_gpg_from_line(line: str) -> t.Tuple[str, str]: + """Returns name and expiration status.""" + assert 'signature from' in line + + name_end = line.split(' from ')[-1] + m = re.search(r'(?P".+") \[(?P\w+)\]', name_end) + assert m + (name, status) = m.groups() + name = name.strip('"\'') + + return (name, status) + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + eq = contents1 == contents2 + + if not eq: + with open(filename1, 'r', encoding='utf-8') as f1, \ + open(filename2, 'r', encoding='utf-8') as f2: + f1lines = f1.readlines() + f2lines = f2.readlines() + + diff = indent( + ''.join(difflib.unified_diff(f1lines, f2lines))) + log.warning(f"found diff in files ({filename1}, {filename2}):\n{diff}\n") + + return eq + + +def get_files_from_hosts_and_compare( + hosts: t.List[str], path: str, filename: str, require_all: bool = False +) -> ReturnCode: + """ + Retrieve the same file from a number of hosts and ensure they have the same contents. + The first host given will be treated as the "primary" host, and is required to succeed. + + Args: + filename: for writing the file locally. + """ + assert len(hosts) > 1 + primary_host = hosts[0] + other_hosts = hosts[1:] + got_files = [] + + def join_url(host: str) -> str: + return host.rstrip('/') + '/' + path.lstrip('/') + + url = join_url(primary_host) + success, output = download_with_wget(url, filename) + if not success: + log.error( + f"couldn't fetch file ({url}). " + "Have you specified the version number in the following format?\n" + f"[{VERSIONPREFIX}]{VERSION_FORMAT} " + f"(example: {VERSIONPREFIX}{VERSION_EXAMPLE})\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_GET_FAILED + else: + log.info(f"got file {url} as {filename}") + got_files.append(filename) + + for i, host in enumerate(other_hosts): + url = join_url(host) + fname = filename + f'.{i + 2}' + success, output = download_with_wget(url, fname) + + if require_all and not success: + log.error( + f"{host} failed to provide file ({url}), but {primary_host} did?\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_MISSING_FROM_ONE_HOST + elif not success: + log.warning( + f"{host} failed to provide file ({url}). " + f"Continuing based solely upon {primary_host}.") + else: + log.info(f"got file {url} as {fname}") + got_files.append(fname) + + for i, got_file in enumerate(got_files): + if got_file == got_files[-1]: + break # break on last file, nothing after it to compare to + + compare_to = got_files[i + 1] + if not files_are_equal(got_file, compare_to): + log.error(f"files not equal: {got_file} and {compare_to}") + return ReturnCode.FILES_NOT_EQUAL + + return ReturnCode.SUCCESS + + +def check_multisig(sigfilename: str, args: argparse.Namespace): + # check signature + # + # We don't write output to a file because this command will almost certainly + # fail with GPG exit code '2' (and so not writing to --output) because of the + # likely presence of multiple untrusted signatures. + retval, output = verify_with_gpg(sigfilename) + + if args.verbose: + log.info(f"gpg output:\n{indent(output)}") + + good, unknown, bad = parse_gpg_result(output.splitlines()) + + if unknown and args.import_keys: + # Retrieve unknown keys and then try GPG again. + for unsig in unknown: + if prompt_yn(f" ? Retrieve key {unsig.key} ({unsig.name})? (y/N) "): + ran = subprocess.run( + ["gpg", "--keyserver", args.keyserver, "--recv-keys", unsig.key]) + + if ran.returncode != 0: + log.warning(f"failed to retrieve key {unsig.key}") + + # Reparse the GPG output now that we have more keys + retval, output = verify_with_gpg(sigfilename) + good, unknown, bad = parse_gpg_result(output.splitlines()) + + return retval, output, good, unknown, bad + + +def prompt_yn(prompt) -> bool: + """Return true if the user inputs 'y'.""" + got = '' + while got not in ['y', 'n']: + got = input(prompt).lower() + return got == 'y' + + def main(args): - # sanity check - if len(args) < 1: - print("Error: need to specify a version on the command line") - return 3 + args = parser.parse_args() + if args.quiet: + log.setLevel(logging.WARNING) + + WORKINGDIR = Path(tempfile.gettempdir()) / f"bitcoin_verify_binaries.{args.version}" + + def cleanup(): + log.info("cleaning up files") + os.chdir(Path.home()) + shutil.rmtree(WORKINGDIR) # determine remote dir dependent on provided version string - version_base, version_rc, os_filter = parse_version_string(args[0]) + try: + version_base, version_rc, os_filter = parse_version_string(args.version) + version_tuple = [int(i) for i in version_base.split('.')] + except Exception as e: + log.debug(e) + log.error(f"unable to parse version; expected format is {VERSION_FORMAT}") + log.error(f" e.g. {VERSION_EXAMPLE}") + return ReturnCode.BAD_VERSION + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" if version_rc: remote_dir += f"test.{version_rc}/" - remote_sigfile = remote_dir + SIGNATUREFILENAME + remote_sigs_path = remote_dir + SIGNATUREFILENAME + remote_sums_path = remote_dir + SUMS_FILENAME # create working directory os.makedirs(WORKINGDIR, exist_ok=True) os.chdir(WORKINGDIR) - # fetch first signature file - sigfile1 = SIGNATUREFILENAME - success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1) - if not success: - print("Error: couldn't fetch signature file. " - "Have you specified the version number in the following format?") - print(f"[{VERSIONPREFIX}][-rc[0-9]][-platform] " - f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)") - print("wget output:") - print(indent(output, '\t')) - return 4 + hosts = [HOST1, HOST2] - # fetch second signature file - sigfile2 = SIGNATUREFILENAME + ".2" - success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2) - if not success: - print("bitcoin.org failed to provide signature file, " - "but bitcoincore.org did?") - print("wget output:") - print(indent(output, '\t')) - remove_files([sigfile1]) - return 5 + got_sig_status = get_files_from_hosts_and_compare( + hosts, remote_sigs_path, SIGNATUREFILENAME, args.require_all_hosts) + if got_sig_status != ReturnCode.SUCCESS: + return got_sig_status - # ensure that both signature files are equal - if not files_are_equal(sigfile1, sigfile2): - print("bitcoin.org and bitcoincore.org signature files were not equal?") - print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}") - return 6 + # Multi-sig verification is available after 22.0. + if version_tuple[0] >= 22: + min_good_sigs = args.min_good_sigs + gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. - # check signature and extract data into file - retval, output = verify_with_gpg(sigfile1, HASHFILE) - if retval != 0: - if retval == 1: - print("Bad signature.") - elif retval == 2: - print("gpg error. Do you have the Bitcoin Core binary release " - "signing key installed?") - print("gpg output:") - print(indent(output, '\t')) - remove_files([sigfile1, sigfile2, HASHFILE]) - return 1 + got_sums_status = get_files_from_hosts_and_compare( + hosts, remote_sums_path, SUMS_FILENAME, args.require_all_hosts) + if got_sums_status != ReturnCode.SUCCESS: + return got_sums_status + + gpg_retval, gpg_output, good, unknown, bad = check_multisig(SIGNATUREFILENAME, args) + else: + log.error("Version too old - single sig not supported. Use a previous " + "version of this script from the repo.") + return ReturnCode.BAD_VERSION + + if gpg_retval not in gpg_allowed_codes: + if gpg_retval == 1: + log.critical(f"Bad signature (code: {gpg_retval}).") + if gpg_retval == 2: + log.critical( + "gpg error. Do you have the Bitcoin Core binary release " + "signing key installed?") + else: + log.critical(f"unexpected GPG exit code ({gpg_retval})") + + log.error(f"gpg output:\n{indent(gpg_output)}") + cleanup() + return ReturnCode.INTEGRITY_FAILURE + + # Decide which keys we trust, though not "trust" in the GPG sense, but rather + # which pubkeys convince us that this sums file is legitimate. In other words, + # which pubkeys within the Bitcoin community do we trust for the purposes of + # binary verification? + trusted_keys = set() + if args.trusted_keys: + trusted_keys |= set(args.trusted_keys.split(',')) + + # Tally signatures and make sure we have enough goods to fulfill + # our threshold. + good_trusted = {sig for sig in good if sig.trusted or sig.key in trusted_keys} + good_untrusted = [sig for sig in good if sig not in good_trusted] + num_trusted = len(good_trusted) + len(good_untrusted) + log.info(f"got {num_trusted} good signatures") + + if num_trusted < min_good_sigs: + log.info("Maybe you need to import " + f"(`gpg --keyserver {args.keyserver} --recv-keys `) " + "some of the following keys: ") + log.info('') + for sig in unknown: + log.info(f" {sig.key} ({sig.name})") + log.info('') + log.error( + "not enough trusted sigs to meet threshold " + f"({num_trusted} vs. {min_good_sigs})") + + return ReturnCode.NOT_ENOUGH_GOOD_SIGS + + for sig in good_trusted: + log.info(f"GOOD SIGNATURE: {sig}") + + for sig in good_untrusted: + log.info(f"GOOD SIGNATURE (untrusted): {sig}") + + for sig in [sig for sig in good if sig.status == 'expired']: + log.warning(f"key {sig.key} for {sig.name} is expired") + + for sig in bad: + log.warning(f"BAD SIGNATURE: {sig}") + + for sig in unknown: + log.warning(f"UNKNOWN SIGNATURE: {sig}") # extract hashes/filenames of binaries to verify from hash file; # each line has the following format: " " - with open(HASHFILE, 'r', encoding='utf8') as hash_file: - hashes_to_verify = [ - line.split()[:2] for line in hash_file if os_filter in line] - remove_files([HASHFILE]) + with open(SUMS_FILENAME, 'r', encoding='utf8') as hash_file: + hashes_to_verify = [line.split()[:2] for line in hash_file if os_filter in line] + remove_files([SUMS_FILENAME]) if not hashes_to_verify: - print("error: no files matched the platform specified") - return 7 + log.error("no files matched the platform specified") + return ReturnCode.NO_BINARIES_MATCH + + # remove binaries that are known not to be hosted by bitcoincore.org + fragments_to_remove = ['-unsigned', '-debug', '-codesignatures'] + for fragment in fragments_to_remove: + nobinaries = [i for i in hashes_to_verify if fragment in i[1]] + if nobinaries: + remove_str = ', '.join(i[1] for i in nobinaries) + log.info( + f"removing *{fragment} binaries ({remove_str}) from verification " + f"since {HOST1} does not host *{fragment} binaries") + hashes_to_verify = [i for i in hashes_to_verify if fragment not in i[1]] # download binaries for _, binary_filename in hashes_to_verify: - print(f"Downloading {binary_filename}") - download_with_wget(HOST1 + remote_dir + binary_filename) + log.info(f"downloading {binary_filename}") + success, output = download_with_wget( + HOST1 + remote_dir + binary_filename, binary_filename) + + if not success: + log.error( + f"failed to download {binary_filename}\n" + f"wget output:\n{indent(output)}") + return ReturnCode.BINARY_DOWNLOAD_FAILED # verify hashes offending_files = [] + files_to_hashes = {} + for hash_expected, binary_filename in hashes_to_verify: with open(binary_filename, 'rb') as binary_file: hash_calculated = sha256(binary_file.read()).hexdigest() if hash_calculated != hash_expected: offending_files.append(binary_filename) + else: + files_to_hashes[binary_filename] = hash_calculated + if offending_files: - print("Hashes don't match.") - print("Offending files:") - print('\n'.join(offending_files)) - return 1 - verified_binaries = [entry[1] for entry in hashes_to_verify] + joined_files = '\n'.join(offending_files) + log.critical( + "Hashes don't match.\n" + f"Offending files:\n{joined_files}") + return ReturnCode.INTEGRITY_FAILURE - # clean up files if desired - if len(args) >= 2: - print("Clean up the binaries") - remove_files([sigfile1, sigfile2] + verified_binaries) + if args.cleanup: + cleanup() else: - print(f"Keep the binaries in {WORKINGDIR}") + log.info(f"did not clean up {WORKINGDIR}") - print("Verified hashes of") - print('\n'.join(verified_binaries)) - return 0 + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + + return ReturnCode.SUCCESS if __name__ == '__main__': From 17575c0efa960ffb765392e3565b3861846f398e Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 22 Mar 2023 21:15:06 -0400 Subject: [PATCH 02/14] contrib: Refactor verifbinaries to support subcommands Prepares for the option to provide local binaries, sha256sums, and signatures directly. --- contrib/verifybinaries/test.py | 15 +- contrib/verifybinaries/verify.py | 329 +++++++++++++++++-------------- 2 files changed, 191 insertions(+), 153 deletions(-) diff --git a/contrib/verifybinaries/test.py b/contrib/verifybinaries/test.py index 539dff658a7..18047e4ebf1 100755 --- a/contrib/verifybinaries/test.py +++ b/contrib/verifybinaries/test.py @@ -8,12 +8,12 @@ from pathlib import Path def main(): """Tests ordered roughly from faster to slower.""" - expect_code(run_verify('0.32'), 4, "Nonexistent version should fail") - expect_code(run_verify('0.32.awefa.12f9h'), 11, "Malformed version should fail") - expect_code(run_verify('22.0 --min-good-sigs 20'), 9, "--min-good-sigs 20 should fail") + expect_code(run_verify("", "pub", '0.32'), 4, "Nonexistent version should fail") + expect_code(run_verify("", "pub", '0.32.awefa.12f9h'), 11, "Malformed version should fail") + expect_code(run_verify('--min-good-sigs 20', "pub", "22.0"), 9, "--min-good-sigs 20 should fail") print("- testing multisig verification (22.0)", flush=True) - _220 = run_verify('22.0 --json') + _220 = run_verify("--json", "pub", "22.0") try: result = json.loads(_220.stdout.decode()) except Exception: @@ -29,12 +29,15 @@ def main(): assert v['bitcoin-22.0-x86_64-linux-gnu.tar.gz'] == '59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16' -def run_verify(extra: str) -> subprocess.CompletedProcess: +def run_verify(global_args: str, command: str, command_args: str) -> subprocess.CompletedProcess: maybe_here = Path.cwd() / 'verify.py' path = maybe_here if maybe_here.exists() else Path.cwd() / 'contrib' / 'verifybinaries' / 'verify.py' + if command == "pub": + command += " --cleanup" + return subprocess.run( - f"{path} --cleanup {extra}", + f"{path} {global_args} {command} {command_args}", stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 6b5fee43ee7..089217a56d2 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -98,60 +98,6 @@ def bool_from_env(key, default=False) -> bool: VERSION_FORMAT = ".[.][-rc[0-9]][-platform]" VERSION_EXAMPLE = "22.0-x86_64 or 0.21.0-rc2-osx" -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument( - 'version', type=str, help=( - f'version of the bitcoin release to download; of the format ' - f'{VERSION_FORMAT}. Example: {VERSION_EXAMPLE}') -) -parser.add_argument( - '-v', '--verbose', action='store_true', - default=bool_from_env('BINVERIFY_VERBOSE'), -) -parser.add_argument( - '-q', '--quiet', action='store_true', - default=bool_from_env('BINVERIFY_QUIET'), -) -parser.add_argument( - '--cleanup', action='store_true', - default=bool_from_env('BINVERIFY_CLEANUP'), - help='if specified, clean up files afterwards' -) -parser.add_argument( - '--import-keys', action='store_true', - default=bool_from_env('BINVERIFY_IMPORTKEYS'), - help='if specified, ask to import each unknown builder key' -) -parser.add_argument( - '--require-all-hosts', action='store_true', - default=bool_from_env('BINVERIFY_REQUIRE_ALL_HOSTS'), - help=( - f'If set, require all hosts ({HOST1}, {HOST2}) to provide signatures. ' - '(Sometimes bitcoin.org lags behind bitcoincore.org.)') -) -parser.add_argument( - '--min-good-sigs', type=int, action='store', nargs='?', - default=int(os.environ.get('BINVERIFY_MIN_GOOD_SIGS', 3)), - help=( - 'The minimum number of good signatures to require successful termination.'), -) -parser.add_argument( - '--keyserver', action='store', nargs='?', - default=os.environ.get('BINVERIFY_KEYSERVER', 'hkp://keyserver.ubuntu.com'), - help='which keyserver to use', -) -parser.add_argument( - '--trusted-keys', action='store', nargs='?', - default=os.environ.get('BINVERIFY_TRUSTED_KEYS', ''), - help='A list of trusted signer GPG keys, separated by commas. Not "trusted keys" in the GPG sense.', -) -parser.add_argument( - '--json', action='store_true', - default=bool_from_env('BINVERIFY_JSON'), - help='If set, output the result as JSON', -) - - def parse_version_string(version_str): if version_str.startswith(VERSIONPREFIX): # remove version prefix version_str = version_str[len(VERSIONPREFIX):] @@ -386,7 +332,7 @@ def get_files_from_hosts_and_compare( return ReturnCode.SUCCESS -def check_multisig(sigfilename: str, args: argparse.Namespace): +def check_multisig(sigfilename: Path, args: argparse.Namespace) -> t.Tuple[int, str, t.List[SigData], t.List[SigData], t.List[SigData]]: # check signature # # We don't write output to a file because this command will almost certainly @@ -423,12 +369,106 @@ def prompt_yn(prompt) -> bool: got = input(prompt).lower() return got == 'y' +def verify_shasums_signature( + signature_file_path: str, sums_file_path: str, args: argparse.Namespace +) -> t.Tuple[ + ReturnCode, t.List[SigData], t.List[SigData], t.List[SigData], t.List[SigData] +]: + min_good_sigs = args.min_good_sigs + gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. -def main(args): - args = parser.parse_args() - if args.quiet: - log.setLevel(logging.WARNING) + gpg_retval, gpg_output, good, unknown, bad = check_multisig(signature_file_path, args) + if gpg_retval not in gpg_allowed_codes: + if gpg_retval == 1: + log.critical(f"Bad signature (code: {gpg_retval}).") + if gpg_retval == 2: + log.critical( + "gpg error. Do you have the Bitcoin Core binary release " + "signing key installed?") + else: + log.critical(f"unexpected GPG exit code ({gpg_retval})") + + log.error(f"gpg output:\n{indent(gpg_output)}") + return (ReturnCode.INTEGRITY_FAILURE, [], [], [], []) + + # Decide which keys we trust, though not "trust" in the GPG sense, but rather + # which pubkeys convince us that this sums file is legitimate. In other words, + # which pubkeys within the Bitcoin community do we trust for the purposes of + # binary verification? + trusted_keys = set() + if args.trusted_keys: + trusted_keys |= set(args.trusted_keys.split(',')) + + # Tally signatures and make sure we have enough goods to fulfill + # our threshold. + good_trusted = [sig for sig in good if sig.trusted or sig.key in trusted_keys] + good_untrusted = [sig for sig in good if sig not in good_trusted] + num_trusted = len(good_trusted) + len(good_untrusted) + log.info(f"got {num_trusted} good signatures") + + if num_trusted < min_good_sigs: + log.info("Maybe you need to import " + f"(`gpg --keyserver {args.keyserver} --recv-keys `) " + "some of the following keys: ") + log.info('') + for sig in unknown: + log.info(f" {sig.key} ({sig.name})") + log.info('') + log.error( + "not enough trusted sigs to meet threshold " + f"({num_trusted} vs. {min_good_sigs})") + + return (ReturnCode.NOT_ENOUGH_GOOD_SIGS, [], [], [], []) + + for sig in good_trusted: + log.info(f"GOOD SIGNATURE: {sig}") + + for sig in good_untrusted: + log.info(f"GOOD SIGNATURE (untrusted): {sig}") + + for sig in [sig for sig in good if sig.status == 'expired']: + log.warning(f"key {sig.key} for {sig.name} is expired") + + for sig in bad: + log.warning(f"BAD SIGNATURE: {sig}") + + for sig in unknown: + log.warning(f"UNKNOWN SIGNATURE: {sig}") + + return (ReturnCode.SUCCESS, good_trusted, good_untrusted, unknown, bad) + + +def parse_sums_file(sums_file_path: Path, filename_filter: str) -> t.List[t.List[str]]: + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: " " + with open(sums_file_path, 'r', encoding='utf8') as hash_file: + return [line.split()[:2] for line in hash_file if filename_filter in line] + + +def verify_binary_hashes(hashes_to_verify: t.List[t.List[str]]) -> t.Tuple[ReturnCode, t.Dict[str, str]]: + offending_files = [] + files_to_hashes = {} + + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + else: + files_to_hashes[binary_filename] = hash_calculated + + if offending_files: + joined_files = '\n'.join(offending_files) + log.critical( + "Hashes don't match.\n" + f"Offending files:\n{joined_files}") + return (ReturnCode.INTEGRITY_FAILURE, files_to_hashes) + + return (ReturnCode.SUCCESS, files_to_hashes) + + +def verify_published_handler(args: argparse.Namespace) -> ReturnCode: WORKINGDIR = Path(tempfile.gettempdir()) / f"bitcoin_verify_binaries.{args.version}" def cleanup(): @@ -464,83 +504,25 @@ def main(args): return got_sig_status # Multi-sig verification is available after 22.0. - if version_tuple[0] >= 22: - min_good_sigs = args.min_good_sigs - gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. - - got_sums_status = get_files_from_hosts_and_compare( - hosts, remote_sums_path, SUMS_FILENAME, args.require_all_hosts) - if got_sums_status != ReturnCode.SUCCESS: - return got_sums_status - - gpg_retval, gpg_output, good, unknown, bad = check_multisig(SIGNATUREFILENAME, args) - else: + if version_tuple[0] < 22: log.error("Version too old - single sig not supported. Use a previous " "version of this script from the repo.") return ReturnCode.BAD_VERSION - if gpg_retval not in gpg_allowed_codes: - if gpg_retval == 1: - log.critical(f"Bad signature (code: {gpg_retval}).") - if gpg_retval == 2: - log.critical( - "gpg error. Do you have the Bitcoin Core binary release " - "signing key installed?") - else: - log.critical(f"unexpected GPG exit code ({gpg_retval})") + got_sums_status = get_files_from_hosts_and_compare( + hosts, remote_sums_path, SUMS_FILENAME, args.require_all_hosts) + if got_sums_status != ReturnCode.SUCCESS: + return got_sums_status - log.error(f"gpg output:\n{indent(gpg_output)}") - cleanup() - return ReturnCode.INTEGRITY_FAILURE + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(SIGNATUREFILENAME, SUMS_FILENAME, args) + if sigs_status != ReturnCode.SUCCESS: + if sigs_status == ReturnCode.INTEGRITY_FAILURE: + cleanup() + return sigs_status - # Decide which keys we trust, though not "trust" in the GPG sense, but rather - # which pubkeys convince us that this sums file is legitimate. In other words, - # which pubkeys within the Bitcoin community do we trust for the purposes of - # binary verification? - trusted_keys = set() - if args.trusted_keys: - trusted_keys |= set(args.trusted_keys.split(',')) - - # Tally signatures and make sure we have enough goods to fulfill - # our threshold. - good_trusted = {sig for sig in good if sig.trusted or sig.key in trusted_keys} - good_untrusted = [sig for sig in good if sig not in good_trusted] - num_trusted = len(good_trusted) + len(good_untrusted) - log.info(f"got {num_trusted} good signatures") - - if num_trusted < min_good_sigs: - log.info("Maybe you need to import " - f"(`gpg --keyserver {args.keyserver} --recv-keys `) " - "some of the following keys: ") - log.info('') - for sig in unknown: - log.info(f" {sig.key} ({sig.name})") - log.info('') - log.error( - "not enough trusted sigs to meet threshold " - f"({num_trusted} vs. {min_good_sigs})") - - return ReturnCode.NOT_ENOUGH_GOOD_SIGS - - for sig in good_trusted: - log.info(f"GOOD SIGNATURE: {sig}") - - for sig in good_untrusted: - log.info(f"GOOD SIGNATURE (untrusted): {sig}") - - for sig in [sig for sig in good if sig.status == 'expired']: - log.warning(f"key {sig.key} for {sig.name} is expired") - - for sig in bad: - log.warning(f"BAD SIGNATURE: {sig}") - - for sig in unknown: - log.warning(f"UNKNOWN SIGNATURE: {sig}") - - # extract hashes/filenames of binaries to verify from hash file; - # each line has the following format: " " - with open(SUMS_FILENAME, 'r', encoding='utf8') as hash_file: - hashes_to_verify = [line.split()[:2] for line in hash_file if os_filter in line] + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(SUMS_FILENAME, os_filter) remove_files([SUMS_FILENAME]) if not hashes_to_verify: log.error("no files matched the platform specified") @@ -570,23 +552,10 @@ def main(args): return ReturnCode.BINARY_DOWNLOAD_FAILED # verify hashes - offending_files = [] - files_to_hashes = {} + hashes_status, files_to_hashes = verify_binary_hashes(hashes_to_verify) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status - for hash_expected, binary_filename in hashes_to_verify: - with open(binary_filename, 'rb') as binary_file: - hash_calculated = sha256(binary_file.read()).hexdigest() - if hash_calculated != hash_expected: - offending_files.append(binary_filename) - else: - files_to_hashes[binary_filename] = hash_calculated - - if offending_files: - joined_files = '\n'.join(offending_files) - log.critical( - "Hashes don't match.\n" - f"Offending files:\n{joined_files}") - return ReturnCode.INTEGRITY_FAILURE if args.cleanup: cleanup() @@ -609,5 +578,71 @@ def main(args): return ReturnCode.SUCCESS +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '-v', '--verbose', action='store_true', + default=bool_from_env('BINVERIFY_VERBOSE'), + ) + parser.add_argument( + '-q', '--quiet', action='store_true', + default=bool_from_env('BINVERIFY_QUIET'), + ) + parser.add_argument( + '--import-keys', action='store_true', + default=bool_from_env('BINVERIFY_IMPORTKEYS'), + help='if specified, ask to import each unknown builder key' + ) + parser.add_argument( + '--min-good-sigs', type=int, action='store', nargs='?', + default=int(os.environ.get('BINVERIFY_MIN_GOOD_SIGS', 3)), + help=( + 'The minimum number of good signatures to require successful termination.'), + ) + parser.add_argument( + '--keyserver', action='store', nargs='?', + default=os.environ.get('BINVERIFY_KEYSERVER', 'hkp://keyserver.ubuntu.com'), + help='which keyserver to use', + ) + parser.add_argument( + '--trusted-keys', action='store', nargs='?', + default=os.environ.get('BINVERIFY_TRUSTED_KEYS', ''), + help='A list of trusted signer GPG keys, separated by commas. Not "trusted keys" in the GPG sense.', + ) + parser.add_argument( + '--json', action='store_true', + default=bool_from_env('BINVERIFY_JSON'), + help='If set, output the result as JSON', + ) + + subparsers = parser.add_subparsers(title="Commands", required=True, dest="command") + + pub_parser = subparsers.add_parser("pub", help="Verify a published release.") + pub_parser.set_defaults(func=verify_published_handler) + pub_parser.add_argument( + 'version', type=str, help=( + f'version of the bitcoin release to download; of the format ' + f'{VERSION_FORMAT}. Example: {VERSION_EXAMPLE}') + ) + pub_parser.add_argument( + '--cleanup', action='store_true', + default=bool_from_env('BINVERIFY_CLEANUP'), + help='if specified, clean up files afterwards' + ) + pub_parser.add_argument( + '--require-all-hosts', action='store_true', + default=bool_from_env('BINVERIFY_REQUIRE_ALL_HOSTS'), + help=( + f'If set, require all hosts ({HOST1}, {HOST2}) to provide signatures. ' + '(Sometimes bitcoin.org lags behind bitcoincore.org.)') + ) + + args = parser.parse_args() + if args.quiet: + log.setLevel(logging.WARNING) + + return args.func(args) + + if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + sys.exit(main()) From e4d577822835d4866e2ad046f23ab411b2910d59 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 22 Mar 2023 22:06:31 -0400 Subject: [PATCH 03/14] contrib: Specify to GPG the SHA256SUMS file that is detached signed --- contrib/verifybinaries/verify.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 089217a56d2..4af7e40af20 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -137,12 +137,13 @@ def download_lines_with_urllib(url) -> t.Tuple[bool, t.List[str]]: def verify_with_gpg( + filename, signature_filename, output_filename: t.Optional[str] = None ) -> t.Tuple[int, str]: args = [ - 'gpg', '--yes', '--decrypt', '--verify-options', 'show-primary-uid-only', - '--output', output_filename if output_filename else '', signature_filename] + 'gpg', '--yes', '--verify', '--verify-options', 'show-primary-uid-only', + '--output', output_filename if output_filename else '', signature_filename, filename] env = dict(os.environ, LANGUAGE='en') result = subprocess.run(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env) @@ -332,13 +333,13 @@ def get_files_from_hosts_and_compare( return ReturnCode.SUCCESS -def check_multisig(sigfilename: Path, args: argparse.Namespace) -> t.Tuple[int, str, t.List[SigData], t.List[SigData], t.List[SigData]]: +def check_multisig(sums_file: str, sigfilename: str, args: argparse.Namespace) -> t.Tuple[int, str, t.List[SigData], t.List[SigData], t.List[SigData]]: # check signature # # We don't write output to a file because this command will almost certainly # fail with GPG exit code '2' (and so not writing to --output) because of the # likely presence of multiple untrusted signatures. - retval, output = verify_with_gpg(sigfilename) + retval, output = verify_with_gpg(sums_file, sigfilename) if args.verbose: log.info(f"gpg output:\n{indent(output)}") @@ -356,7 +357,7 @@ def check_multisig(sigfilename: Path, args: argparse.Namespace) -> t.Tuple[int, log.warning(f"failed to retrieve key {unsig.key}") # Reparse the GPG output now that we have more keys - retval, output = verify_with_gpg(sigfilename) + retval, output = verify_with_gpg(sums_file, sigfilename) good, unknown, bad = parse_gpg_result(output.splitlines()) return retval, output, good, unknown, bad @@ -377,7 +378,7 @@ def verify_shasums_signature( min_good_sigs = args.min_good_sigs gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. - gpg_retval, gpg_output, good, unknown, bad = check_multisig(signature_file_path, args) + gpg_retval, gpg_output, good, unknown, bad = check_multisig(sums_file_path, signature_file_path, args) if gpg_retval not in gpg_allowed_codes: if gpg_retval == 1: From 6b2cebfa2f1526f7eae31eb645c71712f0a69e97 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 22 Mar 2023 22:07:50 -0400 Subject: [PATCH 04/14] contrib: Add verifybinaries command for specifying files to verify In addition to verifying the published releases with the `pub` command, the verifybinaries script is updated to take a `bin` command where the user specifies the local files, sums, and sigs to verify. --- contrib/verifybinaries/verify.py | 82 ++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 4af7e40af20..76986aa0907 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -42,7 +42,7 @@ import textwrap import urllib.request import enum from hashlib import sha256 -from pathlib import Path +from pathlib import PurePath # The primary host; this will fail if we can't retrieve files from here. HOST1 = "https://bitcoincore.org" @@ -440,11 +440,11 @@ def verify_shasums_signature( return (ReturnCode.SUCCESS, good_trusted, good_untrusted, unknown, bad) -def parse_sums_file(sums_file_path: Path, filename_filter: str) -> t.List[t.List[str]]: +def parse_sums_file(sums_file_path: str, filename_filter: t.List[str]) -> t.List[t.List[str]]: # extract hashes/filenames of binaries to verify from hash file; # each line has the following format: " " with open(sums_file_path, 'r', encoding='utf8') as hash_file: - return [line.split()[:2] for line in hash_file if filename_filter in line] + return [line.split()[:2] for line in hash_file if len(filename_filter) == 0 or any(f in line for f in filename_filter)] def verify_binary_hashes(hashes_to_verify: t.List[t.List[str]]) -> t.Tuple[ReturnCode, t.Dict[str, str]]: @@ -579,6 +579,73 @@ def verify_published_handler(args: argparse.Namespace) -> ReturnCode: return ReturnCode.SUCCESS +def verify_binaries_handler(args: argparse.Namespace) -> ReturnCode: + binary_to_basename = {} + for file in args.binary: + binary_to_basename[PurePath(file).name] = file + + sums_sig_path = None + if args.sums_sig_file: + sums_sig_path = Path(args.sums_sig_file) + else: + log.info(f"No signature file specified, assuming it is {args.sums_file}.asc") + sums_sig_path = Path(args.sums_file).with_suffix(".asc") + + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(sums_sig_path, args.sums_file, args) + if sigs_status != ReturnCode.SUCCESS: + return sigs_status + + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(args.sums_file, [k for k, n in binary_to_basename.items()]) + if not hashes_to_verify: + log.error(f"No files in {args.sums_file} match the specified binaries") + return ReturnCode.NO_BINARIES_MATCH + + # Make sure all files are accounted for + sums_file_path = Path(args.sums_file) + missing_files = [] + files_to_hash = [] + if len(binary_to_basename) > 0: + for file_hash, file in hashes_to_verify: + files_to_hash.append([file_hash, binary_to_basename[file]]) + del binary_to_basename[file] + if len(binary_to_basename) > 0: + log.error(f"Not all specified binaries are in {args.sums_file}") + return ReturnCode.NO_BINARIES_MATCH + else: + log.info(f"No binaries specified, assuming all files specified in {args.sums_file} are located relatively") + for file_hash, file in hashes_to_verify: + file_path = Path(sums_file_path.parent.joinpath(file)) + if file_path.exists(): + files_to_hash.append([file_hash, str(file_path)]) + else: + missing_files.append(file) + + # verify hashes + hashes_status, files_to_hashes = verify_binary_hashes(files_to_hash) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status + + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + "missing_binaries": missing_files, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + for filename in missing_files: + print(f"MISSING: {filename}") + + return ReturnCode.SUCCESS + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -638,6 +705,15 @@ def main(): '(Sometimes bitcoin.org lags behind bitcoincore.org.)') ) + bin_parser = subparsers.add_parser("bin", help="Verify local binaries.") + bin_parser.set_defaults(func=verify_binaries_handler) + bin_parser.add_argument("--sums-sig-file", "-s", help="Path to the SHA256SUMS.asc file to verify") + bin_parser.add_argument("sums_file", help="Path to the SHA256SUMS file to verify") + bin_parser.add_argument( + "binary", nargs="*", + help="Path to a binary distribution file to verify. Can be specified multiple times for multiple files to verify." + ) + args = parser.parse_args() if args.quiet: log.setLevel(logging.WARNING) From 7a6e7ffd066a42c5fbb7d69effbe074fb982936b Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 23 Mar 2023 00:56:12 -0400 Subject: [PATCH 05/14] contrib: Use machine parseable GPG output in verifybinaries GPG has an option to provide machine parseable output. Use that instead of trying to parse the human readable output. --- contrib/verifybinaries/verify.py | 139 +++++++++++++++---------------- 1 file changed, 67 insertions(+), 72 deletions(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 76986aa0907..83370fbede1 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -42,7 +42,7 @@ import textwrap import urllib.request import enum from hashlib import sha256 -from pathlib import PurePath +from pathlib import PurePath, Path # The primary host; this will fail if we can't retrieve files from here. HOST1 = "https://bitcoincore.org" @@ -141,14 +141,19 @@ def verify_with_gpg( signature_filename, output_filename: t.Optional[str] = None ) -> t.Tuple[int, str]: - args = [ - 'gpg', '--yes', '--verify', '--verify-options', 'show-primary-uid-only', - '--output', output_filename if output_filename else '', signature_filename, filename] + with tempfile.NamedTemporaryFile() as status_file: + args = [ + 'gpg', '--yes', '--verify', '--verify-options', 'show-primary-uid-only', "--status-file", status_file.name, + '--output', output_filename if output_filename else '', signature_filename, filename] - env = dict(os.environ, LANGUAGE='en') - result = subprocess.run(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env) - log.debug(f'Result from GPG ({result.returncode}): {result.stdout}') - return result.returncode, result.stdout.decode().rstrip() + env = dict(os.environ, LANGUAGE='en') + result = subprocess.run(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env) + + gpg_data = status_file.read().decode().rstrip() + + log.debug(f'Result from GPG ({result.returncode}): {result.stdout.decode()}') + log.debug(f"{gpg_data}") + return result.returncode, gpg_data def remove_files(filenames): @@ -158,11 +163,14 @@ def remove_files(filenames): class SigData: """GPG signature data as parsed from GPG stdout.""" - def __init__(self, key: str, name: str, trusted: bool, status: str): - self.key = key - self.name = name - self.trusted = trusted - self.status = status + def __init__(self): + self.key = None + self.name = "" + self.trusted = False + self.status = "" + + def __bool__(self): + return self.key is not None def __repr__(self): return ( @@ -174,60 +182,60 @@ def parse_gpg_result( output: t.List[str] ) -> t.Tuple[t.List[SigData], t.List[SigData], t.List[SigData]]: """Returns good, unknown, and bad signatures from GPG stdout.""" - good_sigs = [] - unknown_sigs = [] - bad_sigs = [] + good_sigs: t.List[SigData] = [] + unknown_sigs: t.List[SigData] = [] + bad_sigs: t.List[SigData] = [] total_resolved_sigs = 0 - curr_key = None # Ensure that all lines we match on include a prefix that prevents malicious input # from fooling the parser. def line_begins_with(patt: str, line: str) -> t.Optional[re.Match]: - return re.match(r'^\s*(gpg:)?(\s+)' + patt, line) + return re.match(r'^(\[GNUPG:\])\s+' + patt, line) - detected_name = '' + curr_sigs = unknown_sigs + curr_sigdata = SigData() - for i, line in enumerate(output): - if line_begins_with(r"using (ECDSA|RSA) key (0x[0-9a-fA-F]{16}|[0-9a-fA-F]{40})$", line): - if curr_key: - raise RuntimeError( - f"WARNING: encountered a new sig without resolving the last ({curr_key}) - " - "this could mean we have encountered a bad signature! check GPG output!") - curr_key = line.split('key ')[-1].strip() - assert len(curr_key) == 40 or (len(curr_key) == 18 and curr_key.startswith('0x')) - - if line_begins_with(r"Can't check signature: No public key$", line): - if not curr_key: - raise RuntimeError("failed to detect signature being resolved") - unknown_sigs.append(SigData(curr_key, detected_name, False, '')) - detected_name = '' - curr_key = None - - if line_begins_with(r'Good signature from (".+")(\s+)(\[.+\])$', line): - if not curr_key: - raise RuntimeError("failed to detect signature being resolved") - name, status = parse_gpg_from_line(line) - - # It's safe to index output[i + 1] because if we saw a good sig, there should - # always be another line - trusted = ( - 'This key is not certified with a trusted signature' not in output[i + 1]) - good_sigs.append(SigData(curr_key, name, trusted, status)) - curr_key = None - - if line_begins_with("issuer ", line): - detected_name = line.split("issuer ")[-1].strip('"') - - if 'bad signature from' in line.lower(): - if not curr_key: - raise RuntimeError("failed to detect signature being resolved") - name, status = parse_gpg_from_line(line) - bad_sigs.append(SigData(curr_key, name, False, status)) - curr_key = None - - # Track total signatures included - if line_begins_with('Signature made ', line): + for line in output: + if line_begins_with(r"NEWSIG(?:\s|$)", line): total_resolved_sigs += 1 + if curr_sigdata: + curr_sigs.append(curr_sigdata) + curr_sigdata = SigData() + newsig_split = line.split() + if len(newsig_split) == 3: + curr_sigdata.name = newsig_split[2] + + elif line_begins_with(r"GOODSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + + elif line_begins_with(r"EXPKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "expired" + + elif line_begins_with(r"REVKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "revoked" + + elif line_begins_with(r"BADSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = bad_sigs + + elif line_begins_with(r"ERRSIG(?:\s|$)", line): + curr_sigdata.key, _, _, _, _, _ = line.split()[2:8] + curr_sigs = unknown_sigs + + elif line_begins_with(r"TRUST_(UNDEFINED|NEVER)(?:\s|$)", line): + curr_sigdata.trusted = False + + elif line_begins_with(r"TRUST_(MARGINAL|FULLY|ULTIMATE)(?:\s|$)", line): + curr_sigdata.trusted = True + + # The last one won't have been added, so add it now + assert curr_sigdata + curr_sigs.append(curr_sigdata) all_found = len(good_sigs + bad_sigs + unknown_sigs) if all_found != total_resolved_sigs: @@ -238,19 +246,6 @@ def parse_gpg_result( return (good_sigs, unknown_sigs, bad_sigs) -def parse_gpg_from_line(line: str) -> t.Tuple[str, str]: - """Returns name and expiration status.""" - assert 'signature from' in line - - name_end = line.split(' from ')[-1] - m = re.search(r'(?P".+") \[(?P\w+)\]', name_end) - assert m - (name, status) = m.groups() - name = name.strip('"\'') - - return (name, status) - - def files_are_equal(filename1, filename2): with open(filename1, 'rb') as file1: contents1 = file1.read() From c44323a71705b6df9aafe90df24072e735a5c2ff Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 4 Apr 2023 19:45:53 +0000 Subject: [PATCH 06/14] verifybinaries: move all current examples to the pub subcommand --- contrib/verifybinaries/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index 1579e69fc72..4f2722b23f4 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -42,36 +42,36 @@ See the `Config` object for various options. Validate releases with default settings: ```sh -./contrib/verifybinaries/verify.py 22.0 -./contrib/verifybinaries/verify.py 22.0-rc2 -./contrib/verifybinaries/verify.py bitcoin-core-23.0 -./contrib/verifybinaries/verify.py bitcoin-core-23.0-rc1 +./contrib/verifybinaries/verify.py pub 22.0 +./contrib/verifybinaries/verify.py pub 22.0-rc2 +./contrib/verifybinaries/verify.py pub bitcoin-core-23.0 +./contrib/verifybinaries/verify.py pub bitcoin-core-23.0-rc1 ``` Get JSON output and don't prompt for user input (no auto key import): ```sh -./contrib/verifybinaries/verify.py 22.0-x86 --json +./contrib/verifybinaries/verify.py --json pub 22.0-x86 ``` Don't trust builder-keys by default, and rely only on local GPG state and manually specified keys, while requiring a threshold of at least 10 trusted signatures: ```sh -./contrib/verifybinaries/verify.py 22.0-x86 \ +./contrib/verifybinaries/verify.py \ --no-trust-builder-keys \ --trusted-keys 74E2DEF5D77260B98BC19438099BAD163C70FBFA,9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ - --min-trusted-sigs 10 + --min-trusted-sigs 10 pub 22.0-x86 ``` If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: ```sh -./contrib/verifybinaries/verify.py bitcoin-core-22.0-osx -./contrib/verifybinaries/verify.py bitcoin-core-22.0-rc2-win64 +./contrib/verifybinaries/verify.py pub bitcoin-core-22.0-osx +./contrib/verifybinaries/verify.py pub bitcoin-core-22.0-rc2-win64 ``` If you do not want to keep the downloaded binaries, specify anything as the second parameter. ```sh -./contrib/verifybinaries/verify.py bitcoin-core-22.0 delete +./contrib/verifybinaries/verify.py pub bitcoin-core-22.0 delete ``` From 6d118302654481927e864a428950960e26eb7f4a Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 4 Apr 2023 20:34:58 +0000 Subject: [PATCH 07/14] verifybinaries: remove awkward bitcoin-core prefix handling --- contrib/verifybinaries/README.md | 8 +++----- contrib/verifybinaries/verify.py | 7 ++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index 4f2722b23f4..c2b8d2f2dc5 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -44,8 +44,6 @@ Validate releases with default settings: ```sh ./contrib/verifybinaries/verify.py pub 22.0 ./contrib/verifybinaries/verify.py pub 22.0-rc2 -./contrib/verifybinaries/verify.py pub bitcoin-core-23.0 -./contrib/verifybinaries/verify.py pub bitcoin-core-23.0-rc1 ``` Get JSON output and don't prompt for user input (no auto key import): @@ -66,12 +64,12 @@ specified keys, while requiring a threshold of at least 10 trusted signatures: If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: ```sh -./contrib/verifybinaries/verify.py pub bitcoin-core-22.0-osx -./contrib/verifybinaries/verify.py pub bitcoin-core-22.0-rc2-win64 +./contrib/verifybinaries/verify.py pub 22.0-osx +./contrib/verifybinaries/verify.py pub 22.0-rc2-win64 ``` If you do not want to keep the downloaded binaries, specify anything as the second parameter. ```sh -./contrib/verifybinaries/verify.py pub bitcoin-core-22.0 delete +./contrib/verifybinaries/verify.py pub 22.0 delete ``` diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 83370fbede1..e5e171d58d1 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -99,9 +99,6 @@ VERSION_FORMAT = ".[.][-rc[0-9]][-platform]" VERSION_EXAMPLE = "22.0-x86_64 or 0.21.0-rc2-osx" def parse_version_string(version_str): - if version_str.startswith(VERSIONPREFIX): # remove version prefix - version_str = version_str[len(VERSIONPREFIX):] - parts = version_str.split('-') version_base = parts[0] version_rc = "" @@ -290,8 +287,8 @@ def get_files_from_hosts_and_compare( log.error( f"couldn't fetch file ({url}). " "Have you specified the version number in the following format?\n" - f"[{VERSIONPREFIX}]{VERSION_FORMAT} " - f"(example: {VERSIONPREFIX}{VERSION_EXAMPLE})\n" + f"{VERSION_FORMAT} " + f"(example: {VERSION_EXAMPLE})\n" f"wget output:\n{indent(output)}") return ReturnCode.FILE_GET_FAILED else: From 46c73b57c69933d7eb52e28595609e793e8eef6e Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 4 Apr 2023 21:09:42 +0000 Subject: [PATCH 08/14] verifybinaries: README cleanups - Use correct name for verify.py - Add usage examples for verifybinaries bin - Document proper use of new cleanup option - Fixup broken example --- contrib/verifybinaries/README.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index c2b8d2f2dc5..19cc34d1da1 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -17,7 +17,7 @@ must obtain that key for your local GPG installation. You can obtain these keys by - through a browser using a key server (e.g. keyserver.ubuntu.com), - manually using the `gpg --keyserver --recv-keys ` command, or - - you can run the packaged `verifybinaries.py ... --import-keys` script to + - you can run the packaged `verify.py ... --import-keys` script to have it automatically retrieve unrecognized keys. #### Usage @@ -52,13 +52,12 @@ Get JSON output and don't prompt for user input (no auto key import): ./contrib/verifybinaries/verify.py --json pub 22.0-x86 ``` -Don't trust builder-keys by default, and rely only on local GPG state and manually -specified keys, while requiring a threshold of at least 10 trusted signatures: +Rely only on local GPG state and manually specified keys, while requiring a +threshold of at least 10 trusted signatures: ```sh ./contrib/verifybinaries/verify.py \ - --no-trust-builder-keys \ --trusted-keys 74E2DEF5D77260B98BC19438099BAD163C70FBFA,9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ - --min-trusted-sigs 10 pub 22.0-x86 + --min-good-sigs 10 pub 22.0-x86 ``` If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: @@ -68,8 +67,22 @@ If you only want to download the binaries of certain platform, add the correspon ./contrib/verifybinaries/verify.py pub 22.0-rc2-win64 ``` -If you do not want to keep the downloaded binaries, specify anything as the second parameter. +If you do not want to keep the downloaded binaries, specify the cleanup option. ```sh -./contrib/verifybinaries/verify.py pub 22.0 delete +./contrib/verifybinaries/verify.py pub --cleanup 22.0 +``` + +Use the bin subcommand to verify all files listed in a local checksum file + +```sh +./contrib/verifybinaries/verify.py bin SHA256SUMS +``` + +Verify only a subset of the files listed in a local checksum file + +```sh +./contrib/verifybinaries/verify.py bin ~/Downloads/SHA256SUMS \ + ~/Downloads/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz \ + ~/Downloads/bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz ``` From 5668c6473a01528ac7d66b325b18b1cd2bd93063 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 4 Apr 2023 20:36:04 +0000 Subject: [PATCH 09/14] verifybinaries: Don't delete shasums file It may be useful for local validation. --- contrib/verifybinaries/verify.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index e5e171d58d1..4917254677c 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -516,7 +516,6 @@ def verify_published_handler(args: argparse.Namespace) -> ReturnCode: # Extract hashes and filenames hashes_to_verify = parse_sums_file(SUMS_FILENAME, os_filter) - remove_files([SUMS_FILENAME]) if not hashes_to_verify: log.error("no files matched the platform specified") return ReturnCode.NO_BINARIES_MATCH From 4e0396835dd933a28446844da294040345f2e6ad Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 4 Apr 2023 21:28:14 +0000 Subject: [PATCH 10/14] verifybinaries: remove unreachable code --- contrib/verifybinaries/verify.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 4917254677c..bb6bf6129dc 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -375,10 +375,6 @@ def verify_shasums_signature( if gpg_retval not in gpg_allowed_codes: if gpg_retval == 1: log.critical(f"Bad signature (code: {gpg_retval}).") - if gpg_retval == 2: - log.critical( - "gpg error. Do you have the Bitcoin Core binary release " - "signing key installed?") else: log.critical(f"unexpected GPG exit code ({gpg_retval})") From 8cdadd17297e5f4487692eae88b1e60a42c8c4b2 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 5 Apr 2023 15:48:09 +0000 Subject: [PATCH 11/14] verifybinaries: use recommended keyserver by default --- contrib/verifybinaries/verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index bb6bf6129dc..073fcd7490b 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -656,7 +656,7 @@ def main(): ) parser.add_argument( '--keyserver', action='store', nargs='?', - default=os.environ.get('BINVERIFY_KEYSERVER', 'hkp://keyserver.ubuntu.com'), + default=os.environ.get('BINVERIFY_KEYSERVER', 'hkps://keys.openpgp.org'), help='which keyserver to use', ) parser.add_argument( From 4b23b488d2c5662215d78e4963ef5a2b86b4e25b Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 6 Apr 2023 15:16:28 -0400 Subject: [PATCH 12/14] verifybinaries: fix OS download filter Co-authored-by: Reproducibility Matters --- contrib/verifybinaries/verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 073fcd7490b..5aea43a7e46 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -511,7 +511,7 @@ def verify_published_handler(args: argparse.Namespace) -> ReturnCode: return sigs_status # Extract hashes and filenames - hashes_to_verify = parse_sums_file(SUMS_FILENAME, os_filter) + hashes_to_verify = parse_sums_file(SUMS_FILENAME, [os_filter]) if not hashes_to_verify: log.error("no files matched the platform specified") return ReturnCode.NO_BINARIES_MATCH From 8a65e5145c4d128bb6c30c94e68434dd482db489 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 6 Apr 2023 19:35:54 +0000 Subject: [PATCH 13/14] verifybinaries: catch the correct exception --- contrib/verifybinaries/verify.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 5aea43a7e46..47cf8616b17 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -40,6 +40,7 @@ import shutil import tempfile import textwrap import urllib.request +import urllib.error import enum from hashlib import sha256 from pathlib import PurePath, Path @@ -126,7 +127,7 @@ def download_lines_with_urllib(url) -> t.Tuple[bool, t.List[str]]: try: return (True, [ line.strip().decode() for line in urllib.request.urlopen(url).readlines()]) - except urllib.request.HTTPError as e: + except urllib.error.HTTPError as e: log.warning(f"HTTP request to {url} failed (HTTPError): {e}") except Exception as e: log.warning(f"HTTP request to {url} failed ({e})") From 754fb6bb8125317575edec7c20b5617ad27a9bdd Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 6 Apr 2023 19:56:50 +0000 Subject: [PATCH 14/14] verifybinaries: fix argument type error pointed out by mypy --- contrib/verifybinaries/verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 47cf8616b17..aeab8ebacbe 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -580,7 +580,7 @@ def verify_binaries_handler(args: argparse.Namespace) -> ReturnCode: sums_sig_path = Path(args.sums_file).with_suffix(".asc") # Verify the signature on the SHA256SUMS file - sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(sums_sig_path, args.sums_file, args) + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(str(sums_sig_path), args.sums_file, args) if sigs_status != ReturnCode.SUCCESS: return sigs_status