From ca5e41afd7e3cb164b7c3ac99c8a338a121f682f Mon Sep 17 00:00:00 2001 From: phobos Date: Sun, 30 Aug 2009 20:40:02 -0400 Subject: [PATCH 01/81] update the README instructions and OS X makefiles --- contrib/polipo/Makefile.osx | 10 +++++++--- contrib/polipo/README | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/polipo/Makefile.osx b/contrib/polipo/Makefile.osx index 8e748e2adb..55ed1c62f8 100644 --- a/contrib/polipo/Makefile.osx +++ b/contrib/polipo/Makefile.osx @@ -30,9 +30,13 @@ FILE_DEFINES = -DLOCAL_ROOT=\"$(LOCAL_ROOT)/\" \ DEFINES = $(FILE_DEFINES) $(PLATFORM_DEFINES) -UNIVERSAL = -O -g -mmacosx-version-min=10.4 -isysroot /Developer/SDKs/MacOSX10.4u.sdk -arch i386 -arch ppc -LDFLAGS = -Wl,-syslibroot,/Developer/SDKs/MacOSX10.4u.sdk -CFLAGS = $(MD5INCLUDES) $(CDEBUGFLAGS) $(DEFINES) $(EXTRA_DEFINES) $(UNIVERSAL) +# Uncomment the UNIVERSAL, LDFLAGS, CFLAGS lines if you want universal binaries, otherwise +# you'll produce a binary only for your architecture and version of OSX +# UNIVERSAL = -O -g -mmacosx-version-min=10.4 -isysroot /Developer/SDKs/MacOSX10.4u.sdk -arch i386 -arch ppc +# LDFLAGS = -Wl,-syslibroot,/Developer/SDKs/MacOSX10.4u.sdk +# CFLAGS = $(MD5INCLUDES) $(CDEBUGFLAGS) $(DEFINES) $(EXTRA_DEFINES) $(UNIVERSAL) +# If you uncommented the above CFLAGS, remove this next one. +CFLAGS = $(MD5INCLUDES) $(CDEBUGFLAGS) $(DEFINES) $(EXTRA_DEFINES) SRCS = util.c event.c io.c chunk.c atom.c object.c log.c diskcache.c main.c \ config.c local.c http.c client.c server.c auth.c tunnel.c \ diff --git a/contrib/polipo/README b/contrib/polipo/README index 038e354413..e05ab0ceec 100644 --- a/contrib/polipo/README +++ b/contrib/polipo/README @@ -1,4 +1,6 @@ Copyright 2007-2008 Andrew Lewman +Copyright 2009 The Tor Project + ---------------- General Comments ---------------- From 8c585cce3987b04335acc6eebaa9ded3216b6e13 Mon Sep 17 00:00:00 2001 From: Nathan Freitas Date: Tue, 29 Sep 2009 00:44:39 -0400 Subject: [PATCH 02/81] Include util.h and log.h as relative paths. This shouldn't be necessary, but apparently the Android cross-compiler doesn't respect -I as well as it should. (-I is supposed to add to the *front* of the search path. Android's gcc wrapper apparently likes to add to the end. This is broken, but we need to work around it.) --- src/common/crypto.c | 4 ++-- src/or/buffers.c | 2 ++ src/or/eventdns.c | 1 + src/or/or.h | 2 +- src/tools/tor-checkkey.c | 2 +- src/tools/tor-gencert.c | 4 ++-- src/tools/tor-resolve.c | 4 ++-- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index 4a61d3faf3..2927aa2b5b 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -50,9 +50,9 @@ #define CRYPTO_PRIVATE #include "crypto.h" -#include "log.h" +#include "../common/log.h" #include "aes.h" -#include "util.h" +#include "../common/util.h" #include "container.h" #include "compat.h" diff --git a/src/or/buffers.c b/src/or/buffers.c index e5123732cf..1a1b2077cc 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -12,6 +12,8 @@ **/ #define BUFFERS_PRIVATE #include "or.h" +#include "../common/util.h" +#include "../common/log.h" #ifdef HAVE_UNISTD_H #include #endif diff --git a/src/or/eventdns.c b/src/or/eventdns.c index edccb4bfa6..83bff671aa 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -31,6 +31,7 @@ */ #include "eventdns_tor.h" +#include "../common/util.h" #include /* #define NDEBUG */ diff --git a/src/or/or.h b/src/or/or.h index 0162e31f01..613de1ab03 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -82,7 +82,7 @@ #include "crypto.h" #include "tortls.h" -#include "log.h" +#include "../common/log.h" #include "compat.h" #include "container.h" #include "util.h" diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index 6416dbfbb3..739f7332df 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -7,7 +7,7 @@ #include #include "crypto.h" #include "log.h" -#include "util.h" +#include "../common/util.h" #include "compat.h" #include #include diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 2ae4cc22ec..9ade76397a 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -27,8 +27,8 @@ #define CRYPTO_PRIVATE #include "compat.h" -#include "util.h" -#include "log.h" +#include "../common/util.h" +#include "../common/log.h" #include "crypto.h" #include "address.h" diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index f12c3d8dd3..fe7f793dbb 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -6,9 +6,9 @@ #include "orconfig.h" #include "compat.h" -#include "util.h" +#include "../common/util.h" #include "address.h" -#include "log.h" +#include "../common/log.h" #include #include From 76d26ae52d655e9fd0549eacbba9c7e2d5c05cc4 Mon Sep 17 00:00:00 2001 From: Nathan Freitas Date: Tue, 29 Sep 2009 00:46:53 -0400 Subject: [PATCH 03/81] Disable OpenSSL engines when building for Android. Apparently the Android developers dumped OpenSSL's support for hardware acceleration in order to save some memory, so you can't build programs using engines on Android. [Patch revised by nickm] --- src/common/crypto.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/common/crypto.c b/src/common/crypto.c index 2927aa2b5b..e9333bca2b 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -62,6 +62,11 @@ #include +#ifdef ANDROID +/* Android's OpenSSL seems to have removed all of its Engine support. */ +#define DISABLE_ENGINES +#endif + #if OPENSSL_VERSION_NUMBER < 0x00908000l /* On OpenSSL versions before 0.9.8, there is no working SHA256 * implementation, so we use Tom St Denis's nice speedy one, slightly adapted @@ -174,6 +179,7 @@ crypto_log_errors(int severity, const char *doing) } } +#ifndef DISABLE_ENGINES /** Log any OpenSSL engines we're using at NOTICE. */ static void log_engine(const char *fn, ENGINE *e) @@ -188,7 +194,9 @@ log_engine(const char *fn, ENGINE *e) log(LOG_INFO, LD_CRYPTO, "Using default implementation for %s", fn); } } +#endif +#ifndef DISABLE_ENGINES /** Try to load an engine in a shared library via fully qualified path. */ static ENGINE * @@ -206,6 +214,7 @@ try_load_engine(const char *path, const char *engine) } return e; } +#endif /** Initialize the crypto library. Return 0 on success, -1 on failure. */ @@ -218,10 +227,17 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) _crypto_global_initialized = 1; setup_openssl_threading(); if (useAccel > 0) { +#ifdef DISABLE_ENGINES + (void)accelName; + (void)accelDir; + log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled."); +#else ENGINE *e = NULL; + log_info(LD_CRYPTO, "Initializing OpenSSL engine support."); ENGINE_load_builtin_engines(); ENGINE_register_all_complete(); + if (accelName) { if (accelDir) { log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\"" @@ -251,6 +267,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); log_engine("3DES", ENGINE_get_cipher_engine(NID_des_ede3_ecb)); log_engine("AES", ENGINE_get_cipher_engine(NID_aes_128_ecb)); +#endif } else { log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } @@ -274,7 +291,11 @@ crypto_global_cleanup(void) EVP_cleanup(); ERR_remove_state(0); ERR_free_strings(); + +#ifndef DISABLE_ENGINES ENGINE_cleanup(); +#endif + CONF_modules_unload(1); CRYPTO_cleanup_all_ex_data(); #ifdef TOR_IS_MULTITHREADED From cfba9c01bf37a3c2f67b18275522df81c081e898 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 29 Sep 2009 00:48:45 -0400 Subject: [PATCH 04/81] Alter keygen function to generate keys of different lengths. --- src/common/crypto.c | 13 +++++++------ src/common/crypto.h | 4 +++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index e9333bca2b..581d1ba5a0 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -337,7 +337,8 @@ _crypto_new_pk_env_evp_pkey(EVP_PKEY *pkey) return _crypto_new_pk_env_rsa(rsa); } -/** Helper, used by tor-checkkey.c. Return the RSA from a crypto_pk_env_t. */ +/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a + * crypto_pk_env_t. */ RSA * _crypto_pk_env_get_rsa(crypto_pk_env_t *env) { @@ -472,11 +473,11 @@ crypto_free_cipher_env(crypto_cipher_env_t *env) /* public key crypto */ -/** Generate a new public/private keypair in env. Return 0 on - * success, -1 on failure. +/** Generate a bits-bit new public/private keypair in env. + * Return 0 on success, -1 on failure. */ int -crypto_pk_generate_key(crypto_pk_env_t *env) +crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits) { tor_assert(env); @@ -484,7 +485,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) RSA_free(env->key); #if OPENSSL_VERSION_NUMBER < 0x00908000l /* In OpenSSL 0.9.7, RSA_generate_key is all we have. */ - env->key = RSA_generate_key(PK_BYTES*8,65537, NULL, NULL); + env->key = RSA_generate_key(bits, 65537, NULL, NULL); #else /* In OpenSSL 0.9.8, RSA_generate_key is deprecated. */ { @@ -497,7 +498,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) r = RSA_new(); if (!r) goto done; - if (RSA_generate_key_ex(r, PK_BYTES*8, e, NULL) == -1) + if (RSA_generate_key_ex(r, bits, e, NULL) == -1) goto done; env->key = r; diff --git a/src/common/crypto.h b/src/common/crypto.h index 515b870f3d..f0958a8073 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -86,7 +86,9 @@ crypto_cipher_env_t *crypto_new_cipher_env(void); void crypto_free_cipher_env(crypto_cipher_env_t *env); /* public key crypto */ -int crypto_pk_generate_key(crypto_pk_env_t *env); +int crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits); +#define crypto_pk_generate_key(env) \ + crypto_pk_generate_key_with_bits((env), (PK_BYTES*8)) int crypto_pk_read_private_key_from_filename(crypto_pk_env_t *env, const char *keyfile); From d4717957646d9a2f97dd3ca6139e13f67b9b5ff0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 29 Sep 2009 00:49:43 -0400 Subject: [PATCH 05/81] Make tor-gencert build on Android Previously, tor-gencert would call RSA_generate_key() directly. This won't work on Android, which removes the (deprecated since OpenSSL 0.9.8) function. We can't call RSA_generate_key_ex() unconditionally either, since that didn't exist before 0.9.8. Instead, we must call our own crypto_pk_generate_key_with_bits, which knows how to call RSA_generate_key or RSA_generate_key_ex as appropriate. [Based on patch by Nathan Freitas] --- src/tools/tor-gencert.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 9ade76397a..04d53be072 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -218,6 +219,20 @@ parse_commandline(int argc, char **argv) return 0; } +static RSA * +generate_key(int bits) +{ + RSA *rsa = NULL; + crypto_pk_env_t *env = crypto_new_pk_env(); + if (crypto_pk_generate_key_with_bits(env,bits)<0) + goto done; + rsa = _crypto_pk_env_get_rsa(env); + rsa = RSAPrivateKey_dup(rsa); + done: + crypto_free_pk_env(env); + return rsa; +} + /** Try to read the identity key from identity_key_file. If no such * file exists and create_identity_key is set, make a new identity key and * store it. Return 0 on success, nonzero on failure. @@ -238,7 +253,7 @@ load_identity_key(void) } log_notice(LD_GENERAL, "Generating %d-bit RSA identity key.", IDENTITY_KEY_BITS); - if (!(key = RSA_generate_key(IDENTITY_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(IDENTITY_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate identity key."); crypto_log_errors(LOG_ERR, "Generating identity key"); return 1; @@ -323,7 +338,7 @@ generate_signing_key(void) RSA *key; log_notice(LD_GENERAL, "Generating %d-bit RSA signing key.", SIGNING_KEY_BITS); - if (!(key = RSA_generate_key(SIGNING_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(SIGNING_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate signing key."); crypto_log_errors(LOG_ERR, "Generating signing key"); return 1; From 1707b338ee481bafdff443f15c65f422e8065518 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 29 Sep 2009 01:01:28 -0400 Subject: [PATCH 06/81] Add changelog entry for Android work. --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index c79c865efa..a354cac78d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ Changes in version 0.2.2.4-alpha - 2009-??-?? can run tests in their own processes, have smarter setup/teardown code, and so on. The unit test code has moved to its own subdirectory, and has been split into multiple modules. + - Numerous fixes from Nathan Freitas so that Tor can build correctly for + Android phones. o Minor bugfixes: - Fix a couple of smaller issues with gathering statistics. Bugfixes From 6f95435f285eccf58616c939785e5783510c14c9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 11 Oct 2009 23:31:54 -0400 Subject: [PATCH 07/81] Move android changelog entry to now-correct Tor version. --- ChangeLog | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 52bcd7e670..87548e81d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,10 @@ Changes in version 0.2.2.5-alpha - 2009-10-11 o New directory authorities: - Move dizum to an alternate IP address. + o Code simplifications and refactorings + - Numerous changes, bugfixes, and workarounds from Nathan Freitas + to help Tor build correctly for Android phones. + Changes in version 0.2.2.4-alpha - 2009-10-10 o Major bugfixes: @@ -46,8 +50,6 @@ Changes in version 0.2.2.4-alpha - 2009-10-10 can run tests in their own processes, have smarter setup/teardown code, and so on. The unit test code has moved to its own subdirectory, and has been split into multiple modules. - - Numerous fixes from Nathan Freitas so that Tor can build correctly for - Android phones. Changes in version 0.2.2.3-alpha - 2009-09-23 From ecae6f764a4ff4477ef5c421106a5fbd9e954a10 Mon Sep 17 00:00:00 2001 From: Andrew Lewman Date: Mon, 12 Oct 2009 08:36:32 -0400 Subject: [PATCH 08/81] update naming --- contrib/osx/package.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/osx/package.sh b/contrib/osx/package.sh index eeb0f95078..488bd27c1b 100644 --- a/contrib/osx/package.sh +++ b/contrib/osx/package.sh @@ -34,9 +34,9 @@ if [ -x /usr/bin/sw_vers ]; then # the OS version OSVER=`/usr/bin/sw_vers | grep ProductVersion | cut -f2 | cut -d"." -f1,2` case "$OSVER" in - "10.6") ARCH="universal";; - "10.5") ARCH="universal";; - "10.4") ARCH="universal";; + "10.6") ARCH="i386";; + "10.5") ARCH="i386";; + "10.4") ARCH="i386";; "10.3") ARCH="ppc";; "10.2") ARCH="ppc";; "10.1") ARCH="ppc";; From c0ad6909247161a4de00051f2b6611be16eaa8e7 Mon Sep 17 00:00:00 2001 From: Andrew Lewman Date: Mon, 12 Oct 2009 08:36:39 -0400 Subject: [PATCH 09/81] updated naming. --- contrib/osx/Tor | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/osx/Tor b/contrib/osx/Tor index 0660fd7c8d..bcddc0c42b 100755 --- a/contrib/osx/Tor +++ b/contrib/osx/Tor @@ -25,9 +25,9 @@ if [ -x /usr/bin/sw_vers ]; then # the OS version OSVER=`/usr/bin/sw_vers | grep ProductVersion | cut -f2 | cut -d"." -f1,2` case "$OSVER" in - "10.6") ARCH="universal";; - "10.5") ARCH="universal";; - "10.4") ARCH="universal";; + "10.6") ARCH="i386";; + "10.5") ARCH="i386";; + "10.4") ARCH="i386";; "10.3") ARCH="ppc";; "10.2") ARCH="ppc";; "10.1") ARCH="ppc";; @@ -37,7 +37,7 @@ else ARCH="unknown" fi -if [ $ARCH != "universal" ]; then +if [ $ARCH != "i386" ]; then export EVENT_NOKQUEUE=1 fi From 27db67898503121a6223c73392dbcbb69edca4cd Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 12 Oct 2009 15:01:10 -0400 Subject: [PATCH 10/81] move nickm's commit to the correct release --- ChangeLog | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 87548e81d0..72f4aa8456 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Changes in version 0.2.2.6-alpha - 2009-10-?? + o Code simplifications and refactorings: + - Numerous changes, bugfixes, and workarounds from Nathan Freitas + to help Tor build correctly for Android phones. + + Changes in version 0.2.2.5-alpha - 2009-10-11 o Major bugfixes: - Make the tarball compile again. Oops. Bugfix on 0.2.2.4-alpha. @@ -5,10 +11,6 @@ Changes in version 0.2.2.5-alpha - 2009-10-11 o New directory authorities: - Move dizum to an alternate IP address. - o Code simplifications and refactorings - - Numerous changes, bugfixes, and workarounds from Nathan Freitas - to help Tor build correctly for Android phones. - Changes in version 0.2.2.4-alpha - 2009-10-10 o Major bugfixes: From 4b55ef26c93c5cadd866ca0be1e65547df7b6031 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 12 Oct 2009 15:28:29 -0400 Subject: [PATCH 11/81] bump to 0.2.2.5-alpha-dev --- configure.in | 2 +- contrib/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 62451e6705..7ffa578d69 100644 --- a/configure.in +++ b/configure.in @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.2.5-alpha) +AM_INIT_AUTOMAKE(tor, 0.2.2.5-alpha-dev) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index 4fe579abe4..f063d3b046 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -9,7 +9,7 @@ !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.2.5-alpha" +!define VERSION "0.2.2.5-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index fcd3be497c..0c4c5ccd6b 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -226,5 +226,5 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.2.5-alpha" +#define VERSION "0.2.2.5-alpha-dev" From 81eee0ecfff3dac1e9438719d2f7dc0ba7e84a71 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Oct 2009 17:54:04 -0400 Subject: [PATCH 12/81] Fix a crash when using evdns from Libevent 2. When we tried to use the deprecated non-threadsafe evdns interfaces in Libevent 2 without using the also-deprecated event_init() interface, Libevent 2 would sensibly crash, since it has no guess where to find the Libevent library. Here we use the evdns_base_*() functions instead if they're present, and fake them if they aren't. --- ChangeLog | 4 ++ src/or/dns.c | 142 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 102 insertions(+), 44 deletions(-) diff --git a/ChangeLog b/ChangeLog index 72f4aa8456..9ed1cbd191 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,10 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? - Numerous changes, bugfixes, and workarounds from Nathan Freitas to help Tor build correctly for Android phones. + o Minor bugfixes: + - Fix a crash bug when trying to initialize the evdns module in + Libevent 2. + Changes in version 0.2.2.5-alpha - 2009-10-11 o Major bugfixes: diff --git a/src/or/dns.c b/src/or/dns.c index c6f9e8b833..6021ca5f48 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -16,7 +16,6 @@ #ifdef HAVE_EVENT2_DNS_H #include #include -#include #else #include #include "eventdns.h" @@ -25,6 +24,33 @@ #endif #endif +#ifndef HAVE_EVENT2_DNS_H +struct evdns_base; +struct evdns_request; +#define evdns_base_new(x,y) tor_malloc(1) +#define evdns_base_clear_nameservers_and_suspend(base) \ + evdns_clear_nameservers_and_suspend() +#define evdns_base_search_clear(base) evdns_search_clear() +#define evdns_base_set_default_outgoing_bind_address(base, a, len) \ + evdns_set_default_outgoing_bind_address((a),(len)) +#define evdns_base_resolv_conf_parse(base, options, fname) \ + evdns_resolv_conf_parse((options), (fname)) +#define evdns_base_count_nameservers(base) \ + evdns_count_nameservers() +#define evdns_base_resume(base) \ + evdns_resume() +#define evdns_base_config_windows_nameservers(base) \ + evdns_config_windows_nameservers() +#define evdns_base_set_option(base, opt, val, flags) \ + evdns_set_option((opt),(val),(flags)) +#define evdns_base_resolve_ipv4(base, addr, options, cb, ptr) \ + ((evdns_resolve_ipv4(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse_ipv6(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse_ipv6(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#endif + /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -38,6 +64,9 @@ #define DNS_RESOLVE_FAILED_PERMANENT 2 #define DNS_RESOLVE_SUCCEEDED 3 +/** Our evdns_base; this structure handles all our name lookups. */ +static struct evdns_base *the_evdns_base = NULL; + /** Have we currently configured nameservers with eventdns? */ static int nameservers_configured = 0; /** Did our most recent attempt to configure nameservers with eventdns fail? */ @@ -221,8 +250,8 @@ dns_reset(void) { or_options_t *options = get_options(); if (! server_mode(options)) { - evdns_clear_nameservers_and_suspend(); - evdns_search_clear(); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); + evdns_base_search_clear(the_evdns_base); nameservers_configured = 0; tor_free(resolv_conf_fname); resolv_conf_mtime = 0; @@ -1118,6 +1147,13 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + #ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; @@ -1133,18 +1169,14 @@ configure_nameservers(int force) log_warn(LD_BUG, "Couldn't convert outbound bind address to sockaddr." " Ignoring."); } else { - evdns_set_default_outgoing_bind_address((struct sockaddr *)&ss, - socklen); + evdns_base_set_default_outgoing_bind_address(the_evdns_base, + (struct sockaddr *)&ss, + socklen); } } } #endif - if (options->ServerDNSRandomizeCase) - evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); - else - evdns_set_option("randomize-case:", "0", DNS_OPTIONS_ALL); - evdns_set_log_fn(evdns_log_cb); if (conf_fname) { if (stat(conf_fname, &st)) { @@ -1158,16 +1190,17 @@ configure_nameservers(int force) return 0; } if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, + DNS_OPTIONS_ALL, conf_fname))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); goto err; } @@ -1175,38 +1208,48 @@ configure_nameservers(int force) resolv_conf_fname = tor_strdup(conf_fname); resolv_conf_mtime = st.st_mtime; if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); } #ifdef MS_WINDOWS else { if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } - if (evdns_config_windows_nameservers()) { + if (evdns_base_config_windows_nameservers(the_evdns_base)) { log_warn(LD_EXIT,"Could not config nameservers."); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any platform nameservers in " "your Windows configuration."); goto err; } if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); tor_free(resolv_conf_fname); resolv_conf_mtime = 0; } #endif - if (evdns_count_nameservers() == 1) { - evdns_set_option("max-timeouts:", "16", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "10", DNS_OPTIONS_ALL); +#define SET(k,v) \ + evdns_base_set_option(the_evdns_base, (k), (v), DNS_OPTIONS_ALL) + + if (evdns_base_count_nameservers(the_evdns_base) == 1) { + SET("max-timeouts:", "16"); + SET("timeout:", "10"); } else { - evdns_set_option("max-timeouts:", "3", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "5", DNS_OPTIONS_ALL); + SET("max-timeouts:", "3"); + SET("timeout:", "5"); } + if (options->ServerDNSRandomizeCase) + SET("randomize-case:", "1"); + else + SET("randomize-case:", "0"); + +#undef SET + dns_servers_relaunch_checks(); nameservers_configured = 1; @@ -1304,6 +1347,7 @@ static int launch_resolve(edge_connection_t *exitconn) { char *addr = tor_strdup(exitconn->_base.address); + struct evdns_request *req = NULL; tor_addr_t a; int r; int options = get_options()->ServerDNSSearchDomains ? 0 @@ -1322,25 +1366,28 @@ launch_resolve(edge_connection_t *exitconn) if (r == 0) { log_info(LD_EXIT, "Launching eventdns request for %s", escaped_safe_str(exitconn->_base.address)); - r = evdns_resolve_ipv4(exitconn->_base.address, options, - evdns_callback, addr); + req = evdns_base_resolve_ipv4(the_evdns_base, + exitconn->_base.address, options, + evdns_callback, addr); } else if (r == 1) { log_info(LD_EXIT, "Launching eventdns reverse request for %s", escaped_safe_str(exitconn->_base.address)); if (tor_addr_family(&a) == AF_INET) - r = evdns_resolve_reverse(tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse(the_evdns_base, + tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); else - r = evdns_resolve_reverse_ipv6(tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse_ipv6(the_evdns_base, + tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); } else if (r == -1) { log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here."); } - if (r) { - log_warn(LD_EXIT, "eventdns rejected address %s: error %d.", - escaped_safe_str(addr), r); - r = evdns_err_is_transient(r) ? -2 : -1; + r = 0; + if (!req) { + log_warn(LD_EXIT, "eventdns rejected address %s.", escaped_safe_str(addr)); + r = -1; tor_free(addr); /* There is no evdns request in progress; stop * addr from getting leaked. */ } @@ -1478,17 +1525,19 @@ static void launch_wildcard_check(int min_len, int max_len, const char *suffix) { char *addr; - int r; + struct evdns_request *req; addr = crypto_random_hostname(min_len, max_len, "", suffix); log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent " "domains with request for bogus hostname \"%s\"", addr); - r = evdns_resolve_ipv4(/* This "addr" tells us which address to resolve */ + req = evdns_base_resolve_ipv4( + the_evdns_base, + /* This "addr" tells us which address to resolve */ addr, DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback, /* This "addr" is an argument to the callback*/ addr); - if (r) { + if (!req) { /* There is no evdns request in progress; stop addr from getting leaked */ tor_free(addr); } @@ -1500,6 +1549,7 @@ static void launch_test_addresses(int fd, short event, void *args) { or_options_t *options = get_options(); + struct evdns_request *req; (void)fd; (void)event; (void)args; @@ -1511,14 +1561,18 @@ launch_test_addresses(int fd, short event, void *args) * be an exit server.*/ if (!options->ServerDNSTestAddresses) return; - SMARTLIST_FOREACH(options->ServerDNSTestAddresses, const char *, address, - { - int r = evdns_resolve_ipv4(address, DNS_QUERY_NO_SEARCH, evdns_callback, - tor_strdup(address)); - if (r) - log_info(LD_EXIT, "eventdns rejected test address %s: error %d", - escaped_safe_str(address), r); - }); + SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses, + const char *, address) { + char *a = tor_strdup(address); + req = evdns_base_resolve_ipv4(the_evdns_base, + address, DNS_QUERY_NO_SEARCH, evdns_callback, a); + + if (!req) { + log_info(LD_EXIT, "eventdns rejected test address %s", + escaped_safe_str(address)); + tor_free(a); + } + } SMARTLIST_FOREACH_END(address); } #define N_WILDCARD_CHECKS 2 From 71cdd99dd748e4430572ab9a2e5f2821fb28c480 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Oct 2009 18:57:25 -0400 Subject: [PATCH 13/81] Another event2 evdns fix. --- src/or/dns.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/or/dns.c b/src/or/dns.c index 6021ca5f48..ffd30c89d8 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1622,7 +1622,8 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ if (! launch_event) - launch_event = tor_evtimer_new(NULL, launch_test_addresses, NULL); + launch_event = tor_evtimer_new(tor_libevent_get_base(), + launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; if (evtimer_add(launch_event, &timeout)<0) { From 0a9356c4b13462e00d5fd60f55ddd5cdffeab92c Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 14 Oct 2009 15:49:26 -0400 Subject: [PATCH 14/81] add blurbs for recent dev releases --- ChangeLog | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9ed1cbd191..daa7241885 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,8 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? Changes in version 0.2.2.5-alpha - 2009-10-11 + Tor 0.2.2.5-alpha fixes a few compile problems in 0.2.2.4-alpha. + o Major bugfixes: - Make the tarball compile again. Oops. Bugfix on 0.2.2.4-alpha. @@ -17,6 +19,11 @@ Changes in version 0.2.2.5-alpha - 2009-10-11 Changes in version 0.2.2.4-alpha - 2009-10-10 + Tor 0.2.2.4-alpha fixes more crash bugs in 0.2.2.2-alpha. It also + introduces a new unit test framework, shifts directry authority + addresses around to reduce the impact from recent blocking events, + and fixes a few smaller bugs. + o Major bugfixes: - Fix several more asserts in the circuit_build_times code, for example one that causes Tor to fail to start once we have @@ -42,7 +49,7 @@ Changes in version 0.2.2.4-alpha - 2009-10-10 - Fix a couple of smaller issues with gathering statistics. Bugfixes on 0.2.2.1-alpha. - Fix two memory leaks in the error case of - circuit_build_times_parse_state. Bugfix on 0.2.2.2-alpha. + circuit_build_times_parse_state(). Bugfix on 0.2.2.2-alpha. - Don't count one-hop circuits when we're estimating how long it takes circuits to build on average. Otherwise we'll set our circuit build timeout lower than we should. Bugfix on 0.2.2.2-alpha. @@ -54,7 +61,7 @@ Changes in version 0.2.2.4-alpha - 2009-10-10 o Code simplifications and refactoring: - Revise our unit tests to use the "tinytest" framework, so we can run tests in their own processes, have smarter setup/teardown - code, and so on. The unit test code has moved to its own + code, and so on. The unit test code has moved to its own subdirectory, and has been split into multiple modules. From 83c3f118db0ae3911ea72403856df9fb08b2d0e5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Oct 2009 16:15:41 -0400 Subject: [PATCH 15/81] Code to parse and access network parameters. Partial backport of 381766ce4b1145460. Partial backport of 56c6d78520a98fb64. --- src/common/torint.h | 4 ++++ src/or/networkstatus.c | 31 +++++++++++++++++++++++++++++++ src/or/or.h | 12 +++++++++--- src/or/routerparse.c | 31 +++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/common/torint.h b/src/common/torint.h index 1f7421174a..f8441859a9 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -119,6 +119,10 @@ typedef unsigned int uint32_t; #endif #endif +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif + #if (SIZEOF_LONG == 4) #ifndef HAVE_INT32_T typedef signed long int32_t; diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 573197a53f..05da73b5cb 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -286,6 +286,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c)); smartlist_free(ns->known_flags); } + if (ns->net_params) { + SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c)); + smartlist_free(ns->net_params); + } if (ns->supported_methods) { SMARTLIST_FOREACH(ns->supported_methods, char *, c, tor_free(c)); smartlist_free(ns->supported_methods); @@ -1884,6 +1888,33 @@ networkstatus_dump_bridge_status_to_file(time_t now) tor_free(status); } +/** Return the value of a integer parameter from the networkstatus ns + * whose name is param_name. Return default_val if ns is NULL, + * or if it has no parameter called param_name. */ +int32_t +networkstatus_get_param(networkstatus_t *ns, const char *param_name, + int32_t default_val) +{ + size_t name_len; + + if (!ns || !ns->net_params) + return default_val; + + name_len = strlen(param_name); + + SMARTLIST_FOREACH_BEGIN(ns->net_params, const char *, p) { + if (!strcmpstart(p, param_name) && p[name_len] == '=') { + int ok=0; + long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, INT32_MAX, &ok, + NULL); + if (ok) + return (int32_t) v; + } + } SMARTLIST_FOREACH_END(p); + + return default_val; +} + /** If question is a string beginning with "ns/" in a format the * control interface expects for a GETINFO question, set *answer to a * newly-allocated string containing networkstatus lines for the appropriate diff --git a/src/or/or.h b/src/or/or.h index 319b3a9d10..0c0d8e869e 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1655,6 +1655,10 @@ typedef struct networkstatus_t { * not listed here, the voter has no opinion on what its value should be. */ smartlist_t *known_flags; + /** List of key=value strings for the parameters in this vote or + * consensus, sorted by key. */ + smartlist_t *net_params; + /** List of networkstatus_voter_info_t. For a vote, only one element * is included. For a consensus, one element is included for every voter * whose vote contributed to the consensus. */ @@ -3570,9 +3574,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); #ifdef DIRVOTE_PRIVATE -char * -format_networkstatus_vote(crypto_pk_env_t *private_key, - networkstatus_t *v3_ns); +char *format_networkstatus_vote(crypto_pk_env_t *private_key, + networkstatus_t *v3_ns); +char *dirvote_compute_params(smartlist_t *votes); #endif /********************************* dns.c ***************************/ @@ -3787,6 +3791,8 @@ void signed_descs_update_status_from_consensus_networkstatus( char *networkstatus_getinfo_helper_single(routerstatus_t *rs); char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); void networkstatus_dump_bridge_status_to_file(time_t now); +int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, + int32_t default_val); int getinfo_helper_networkstatus(control_connection_t *conn, const char *question, char **answer); void networkstatus_free_all(void); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 189458ee1e..4e1d0cd592 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -77,6 +77,7 @@ typedef enum { K_VOTING_DELAY, K_KNOWN_FLAGS, + K_PARAMS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, K_CONSENSUS_METHODS, @@ -383,6 +384,7 @@ static token_rule_t networkstatus_token_table[] = { T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("voting-delay", K_VOTING_DELAY, GE(2), NO_OBJ ), T1("known-flags", K_KNOWN_FLAGS, ARGS, NO_OBJ ), + T01("params", K_PARAMS, ARGS, NO_OBJ ), T( "fingerprint", K_FINGERPRINT, CONCAT_ARGS, NO_OBJ ), CERTIFICATE_MEMBERS @@ -420,6 +422,7 @@ static token_rule_t networkstatus_consensus_token_table[] = { T01("client-versions", K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ), T01("server-versions", K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ), T01("consensus-method", K_CONSENSUS_METHOD, EQ(1), NO_OBJ), + T01("params", K_PARAMS, ARGS, NO_OBJ ), END_OF_TABLE }; @@ -2310,6 +2313,34 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } + tok = find_opt_by_keyword(tokens, K_PARAMS); + if (tok) { + inorder = 1; + ns->net_params = smartlist_create(); + for (i = 0; i < tok->n_args; ++i) { + int ok=0; + char *eq = strchr(tok->args[i], '='); + if (!eq) { + log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i])); + goto err; + } + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i])); + goto err; + } + if (i > 0 && strcmp(tok->args[i-1], tok->args[i]) >= 0) { + log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]); + inorder = 0; + } + smartlist_add(ns->net_params, tor_strdup(tok->args[i])); + } + if (!inorder) { + log_warn(LD_DIR, "params not in order"); + goto err; + } + } + ns->voters = smartlist_create(); SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { From 23943364263b8cb38e81a63715f872691269d5ed Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 14 Oct 2009 17:07:32 -0400 Subject: [PATCH 16/81] read the "circwindow" parameter from the consensus backport of c43859c5c12361fad505 backport of 0d13e0ed145f4c1b5bd1 --- ChangeLog | 5 +++++ src/or/circuitbuild.c | 2 +- src/or/circuitlist.c | 15 ++++++++++++++- src/or/networkstatus.c | 8 ++++++-- src/or/or.h | 5 +++-- src/or/rendclient.c | 2 +- src/or/rendservice.c | 2 +- 7 files changed, 31 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1f33eb741f..731a483880 100644 --- a/ChangeLog +++ b/ChangeLog @@ -10,6 +10,11 @@ Changes in version 0.2.1.20 - 2009-??-?? contains more than one signature from the same voter. Bugfix on 0.2.0.3-alpha. + o Major features: + - Tor now reads the "circwindow" parameter out of the consensus, + and uses that value for its circuit package window rather than the + default of 1000 cells. Begins the implementation of proposal 168. + o New directory authorities: - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory authority. diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 983eb6dac1..4b5ba62fa2 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1829,7 +1829,7 @@ onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) hop->extend_info = extend_info_dup(choice); - hop->package_window = CIRCWINDOW_START; + hop->package_window = circuit_initial_package_window(); hop->deliver_window = CIRCWINDOW_START; return 0; diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 252eaf9f8e..5918bdd7ae 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -361,6 +361,19 @@ circuit_purpose_to_controller_string(uint8_t purpose) } } +/** Pick a reasonable package_window to start out for our circuits. + * Originally this was hard-coded at 1000, but now the consensus votes + * on the answer. See proposal 168. */ +int32_t +circuit_initial_package_window(void) +{ + int32_t num = networkstatus_get_param(NULL, "circwindow", CIRCWINDOW_START); + /* If the consensus tells us a negative number, we'd assert. */ + if (num < 0) + num = CIRCWINDOW_START; + return num; +} + /** Initialize the common elements in a circuit_t, and add it to the global * list. */ static void @@ -368,7 +381,7 @@ init_circuit_base(circuit_t *circ) { circ->timestamp_created = time(NULL); - circ->package_window = CIRCWINDOW_START; + circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; circuit_add(circ); diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 05da73b5cb..f4a0761f7b 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1889,14 +1889,18 @@ networkstatus_dump_bridge_status_to_file(time_t now) } /** Return the value of a integer parameter from the networkstatus ns - * whose name is param_name. Return default_val if ns is NULL, - * or if it has no parameter called param_name. */ + * whose name is param_name. If ns is NULL, try loading the + * latest consensus ourselves. Return default_val if no latest + * consensus, or if it has no parameter called param_name. */ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int32_t default_val) { size_t name_len; + if (!ns) /* if they pass in null, go find it ourselves */ + ns = networkstatus_get_latest_consensus(); + if (!ns || !ns->net_params) return default_val; diff --git a/src/or/or.h b/src/or/or.h index 0c0d8e869e..ae65127e36 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1853,9 +1853,9 @@ typedef struct crypt_path_t { struct crypt_path_t *prev; /**< Link to previous crypt_path_t in the * circuit. */ - int package_window; /**< How many bytes are we allowed to originate ending + int package_window; /**< How many cells are we allowed to originate ending * at this step? */ - int deliver_window; /**< How many bytes are we willing to deliver originating + int deliver_window; /**< How many cells are we willing to deliver originating * at this step? */ } crypt_path_t; @@ -2789,6 +2789,7 @@ void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, or_connection_t *conn); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); +int32_t circuit_initial_package_window(void); origin_circuit_t *origin_circuit_new(void); or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 13e43c87b7..47a8818a50 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -703,7 +703,7 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, /* set the windows to default. these are the windows * that alice thinks bob has. */ - hop->package_window = CIRCWINDOW_START; + hop->package_window = circuit_initial_package_window(); hop->deliver_window = CIRCWINDOW_START; onion_append_to_cpath(&circ->cpath, hop); diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 3144ef2f04..d2868b738d 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1556,7 +1556,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) /* set the windows to default. these are the windows * that bob thinks alice has. */ - hop->package_window = CIRCWINDOW_START; + hop->package_window = circuit_initial_package_window(); hop->deliver_window = CIRCWINDOW_START; onion_append_to_cpath(&circuit->cpath, hop); From 2bee297d57c9a345e791f54f0b6f373b53f74bef Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 14 Oct 2009 19:36:08 -0400 Subject: [PATCH 17/81] Move moria1 and Tonga to alternate IP addresses. --- ChangeLog | 1 + src/or/config.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 731a483880..96ffff9b1e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,7 @@ Changes in version 0.2.1.20 - 2009-??-?? o New directory authorities: - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory authority. + - Move moria1 and Tonga to alternate IP addresses. o Minor bugfixes: - Fix a signed/unsigned compile warning in 0.2.1.19. diff --git a/src/or/config.c b/src/or/config.c index 8fd70bec9f..4e2a1765dc 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -904,14 +904,14 @@ add_default_trusted_dir_authorities(authority_type_t type) int i; const char *dirservers[] = { "moria1 v1 orport=9001 v3ident=E2A2AF570166665D738736D0DD58169CC61D8A8B " - "128.31.0.34:9031 FFCB 46DB 1339 DA84 674C 70D7 CB58 6434 C437 0441", + "128.31.0.39:9031 FFCB 46DB 1339 DA84 674C 70D7 CB58 6434 C437 0441", "moria2 v1 orport=9002 128.31.0.34:9032 " "719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF", "tor26 v1 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " "86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D", "dizum orport=443 v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 " "194.109.206.212:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755", - "Tonga orport=443 bridge no-v2 82.94.251.206:80 " + "Tonga orport=443 bridge no-v2 82.94.251.203:80 " "4A0C CD2D DC79 9508 3D73 F5D6 6710 0C8A 5831 F16D", "ides orport=9090 no-v2 v3ident=27B6B5996C426270A5C95488AA5BCEB6BCC86956 " "216.224.124.114:9030 F397 038A DC51 3361 35E7 B80B D99C A384 4360 292B", From 16dc543851bb82c481d4319d557ffea5bef6cc50 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 15 Oct 2009 12:00:35 -0400 Subject: [PATCH 18/81] bump to 0.2.1.20 --- ChangeLog | 10 +++++----- configure.in | 2 +- contrib/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 96ffff9b1e..e8ec768408 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -Changes in version 0.2.1.20 - 2009-??-?? +Changes in version 0.2.1.20 - 2009-10-15 o Major bugfixes: - Send circuit or stream sendme cells when our window has decreased by 100 cells, not when it has decreased by 101 cells. Bug uncovered @@ -9,6 +9,9 @@ Changes in version 0.2.1.20 - 2009-??-?? - Fix a remotely triggerable memory leak when a consensus document contains more than one signature from the same voter. Bugfix on 0.2.0.3-alpha. + - Avoid segfault in rare cases when finishing an introduction circuit + as a client and finding out that we don't have an introduction key + for it. Fixes bug 1073. Reported by Aaron Swartz. o Major features: - Tor now reads the "circwindow" parameter out of the consensus, @@ -18,7 +21,7 @@ Changes in version 0.2.1.20 - 2009-??-?? o New directory authorities: - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory authority. - - Move moria1 and Tonga to alternate IP addresses. + - Move moria1 and tonga to alternate IP addresses. o Minor bugfixes: - Fix a signed/unsigned compile warning in 0.2.1.19. @@ -46,9 +49,6 @@ Changes in version 0.2.1.20 - 2009-??-?? excluded in ExcludeExitNodes, but the circuit is not used to access the outside world. This should help fix bug 1090. Bugfix on 0.2.1.6-alpha. - - Avoid segfault in rare cases when finishing an introduction circuit - as a client and finding out that we don't have an introduction key - for it. Fixes bug 1073. Reported by Aaron Swartz. - Work around a small memory leak in some versions of OpenSSL that stopped the memory used by the hostname TLS extension from being freed. diff --git a/configure.in b/configure.in index 5b30198f22..ce3b0a3be5 100644 --- a/configure.in +++ b/configure.in @@ -5,7 +5,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.1.19) +AM_INIT_AUTOMAKE(tor, 0.2.1.20) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index eccf41984a..7644d24449 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -9,7 +9,7 @@ !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.1.19" +!define VERSION "0.2.1.20" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 03e615850f..dc049bfdc7 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -226,6 +226,6 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.1.19" +#define VERSION "0.2.1.20" From 8d41e6c47126fb467858f98b06ab7460dcc6647b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Aug 2009 20:03:40 -0400 Subject: [PATCH 19/81] Support for encoding and decoding 256-bit digests in base64 --- src/common/crypto.c | 38 ++++++++++++++++++++++++++++++++++++++ src/common/crypto.h | 2 ++ src/test/test_crypto.c | 11 +++++++++++ 3 files changed, 51 insertions(+) diff --git a/src/common/crypto.c b/src/common/crypto.c index 581d1ba5a0..21c8aed2d4 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -2274,6 +2274,44 @@ digest_from_base64(char *digest, const char *d64) #endif } +/** Base-64 encode DIGEST256_LINE bytes from digest, remove the + * trailing = and newline characters, and store the nul-terminated result in + * the first BASE64_DIGEST256_LEN+1 bytes of d64. */ +int +digest256_to_base64(char *d64, const char *digest) +{ + char buf[256]; + base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN); + buf[BASE64_DIGEST256_LEN] = '\0'; + memcpy(d64, buf, BASE64_DIGEST256_LEN+1); + return 0; +} + +/** Given a base-64 encoded, nul-terminated digest in d64 (without + * trailing newline or = characters), decode it and store the result in the + * first DIGEST256_LEN bytes at digest. */ +int +digest256_from_base64(char *digest, const char *d64) +{ +#ifdef USE_OPENSSL_BASE64 + char buf_in[BASE64_DIGEST256_LEN+3]; + char buf[256]; + if (strlen(d64) != BASE64_DIGEST256_LEN) + return -1; + memcpy(buf_in, d64, BASE64_DIGEST256_LEN); + memcpy(buf_in+BASE64_DIGEST256_LEN, "=\n\0", 3); + if (base64_decode(buf, sizeof(buf), buf_in, strlen(buf_in)) != DIGEST256_LEN) + return -1; + memcpy(digest, buf, DIGEST256_LEN); + return 0; +#else + if (base64_decode(digest, DIGEST256_LEN, d64, strlen(d64)) == DIGEST256_LEN) + return 0; + else + return -1; +#endif +} + /** Implements base32 encoding as in rfc3548. Limitation: Requires * that srclen*8 is a multiple of 5. */ diff --git a/src/common/crypto.h b/src/common/crypto.h index f0958a8073..63ea96d056 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -211,6 +211,8 @@ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); int digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); +int digest256_to_base64(char *d64, const char *digest); +int digest256_from_base64(char *digest, const char *d64); /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the * 9th describes how much iteration to do. */ diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index d235520d09..2edb8035bb 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -472,6 +472,17 @@ test_crypto_formats(void) test_assert(digest_from_base64(data3, "###") < 0); + /* Encoding SHA256 */ + crypto_rand(data2, DIGEST256_LEN); + memset(data2, 100, 1024); + digest256_to_base64(data2, data1); + test_eq(BASE64_DIGEST256_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST256_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest256_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST256_LEN); + test_eq(99, data3[DIGEST256_LEN+1]); + /* Base32 tests */ strlcpy(data1, "5chrs", 1024); /* bit pattern is: [35 63 68 72 73] -> From 8b2f6b27fdc03f12d092d37c42d1995ff7426916 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Sep 2009 11:57:19 -0400 Subject: [PATCH 20/81] Make signature-generation code handle different key and digest lengths. --- src/or/dirserv.c | 9 ++++++--- src/or/dirvote.c | 6 +++--- src/or/or.h | 1 + src/or/rendcommon.c | 3 ++- src/or/router.c | 5 +++-- src/or/routerparse.c | 9 +++++---- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 47dc038454..f12ef2f3d5 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1091,7 +1091,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1549,7 +1550,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -2743,7 +2745,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 358246ae9d..4e94eb67e6 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -192,7 +192,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, if (router_get_networkstatus_v3_hash(status, digest)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -1257,7 +1257,7 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, signing_key_fingerprint); /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1272,7 +1272,7 @@ networkstatus_compute_consensus(smartlist_t *votes, signing_key_fingerprint, 0); tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ diff --git a/src/or/or.h b/src/or/or.h index f0ea25e07e..d9e883f3e8 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4911,6 +4911,7 @@ int router_get_networkstatus_v3_hash(const char *s, char *digest); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, + size_t digest_len, crypto_pk_env_t *private_key); int router_parse_list_from_string(const char **s, const char *eos, smartlist_t *dest, diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index df7195e3ea..c7eb2a9d08 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -618,7 +618,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; diff --git a/src/or/router.c b/src/or/router.c index 5b260de1ca..85abc8c5fb 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1788,7 +1788,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1980,7 +1980,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, len += strlen(s+len); if (router_get_extrainfo_hash(s, digest)<0) return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) + if (router_append_dirobj_signature(s+len, maxlen-len, digest, DIGEST_LEN, + ident_key)<0) return -1; { diff --git a/src/or/routerparse.c b/src/or/routerparse.c index e35ece06de..02c5cdb44e 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -643,14 +643,15 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i; + int siglen; signature = tor_malloc(crypto_pk_keysize(private_key)); - if (crypto_pk_private_sign(private_key, signature, digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -658,7 +659,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } From 15f4e9600cb8bb127bcf7004691fe9c59eb7505e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Sep 2009 12:02:00 -0400 Subject: [PATCH 21/81] Signature-checking code can handle longer digests. --- src/or/routerparse.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 02c5cdb44e..f6d6dc9a14 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -528,6 +528,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -802,7 +803,7 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; @@ -895,7 +896,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -996,6 +997,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -1026,14 +1028,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(tok->object_size); if (crypto_pk_public_checksig(pkey, signed_digest, tok->object_body, tok->object_size) - != DIGEST_LEN) { + != digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (memcmp(digest, signed_digest, DIGEST_LEN)) { + if (memcmp(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1489,7 +1491,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1617,7 +1619,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1779,7 +1782,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1788,6 +1791,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -2236,7 +2240,7 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; @@ -2632,7 +2636,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, + tok, ns->cert->signing_key, 0, "network-status vote")) goto err; v->good_signature = 1; @@ -3812,7 +3817,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ From c5f7f04aff850e8e3fad28e93e6300447625fdbb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Sep 2009 12:02:49 -0400 Subject: [PATCH 22/81] Allow signed data to include other hashes later. Previously, we insisted that a valid signature must be a signature of the expected digest. Now we accept anything that starts with the expected digest. This lets us include another digest later. --- src/or/routerparse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/routerparse.c b/src/or/routerparse.c index f6d6dc9a14..bfcf20b09c 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1028,7 +1028,7 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(tok->object_size); if (crypto_pk_public_checksig(pkey, signed_digest, tok->object_body, tok->object_size) - != digest_len) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; From 80a7a34755e7b7163778c5475f9a6459febb1bf4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Sep 2009 13:29:48 -0400 Subject: [PATCH 23/81] Revise 162's idea of how detached signatures work with flavors The original proposal was vague and would have made older Tors reject detached-signature documents as soon as they saw one with flavors. --- doc/spec/proposals/162-consensus-flavors.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index 8fdf9d07bf..56a0b0e1ab 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -91,9 +91,18 @@ Spec modifications: 3. Document format: detached signatures. - In addition to the current detached signature format, we allow - the first line to take the form, - "consensus-digest" SP flavor SP 1*(Algname "=" Digest) NL + We amend the detached signature format to include more than one + consensus-digest line, and more than one set of signatures. + + After the consensus-digest line, we allow more lines of the form: + "additional-digest" SP flavor SP algname SP digest NL + + Before the directory-signature lines, we allow more entries of the form: + "additional-signature" SP flavor SP algname SP identity SP + signing-key-digest NL signature. + + [We do not use "consensus-digest" or "directory-signature" for flavored + consensuses, since this could confuse older Tors.] The consensus-signatures URL should contain the signatures for _all_ flavors of consensus. From bdf48393956db2827eb4340e2484cc28282b3f34 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Aug 2009 20:06:59 -0400 Subject: [PATCH 24/81] Functions to encode microdescriptors and their lines. --- src/or/dirvote.c | 76 +++++++++++++++++++++++++++++++++++++++++++++--- src/or/or.h | 6 ++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 4e94eb67e6..f0f92d2f60 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -21,7 +21,7 @@ static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); static int dirvote_compute_consensus(void); static int dirvote_publish_consensus(void); -static char *make_consensus_method_list(int low, int high); +static char *make_consensus_method_list(int low, int high, const char *sep); /** The highest consensus method that we currently support. */ #define MAX_SUPPORTED_CONSENSUS_METHOD 7 @@ -102,7 +102,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char *params; authority_cert_t *cert = v3_ns->cert; char *methods = - make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD); + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); @@ -480,7 +480,7 @@ consensus_method_is_supported(int method) /** Return a newly allocated string holding the numbers between low and high * (inclusive) that are supported consensus methods. */ static char * -make_consensus_method_list(int low, int high) +make_consensus_method_list(int low, int high, const char *separator) { char *list; @@ -494,7 +494,7 @@ make_consensus_method_list(int low, int high) tor_snprintf(b, sizeof(b), "%d", i); smartlist_add(lst, tor_strdup(b)); } - list = smartlist_join_strings(lst, " ", 0, NULL); + list = smartlist_join_strings(lst, separator, 0, NULL); tor_assert(list); SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); smartlist_free(lst); @@ -2389,3 +2389,71 @@ dirvote_get_vote(const char *fp, int flags) return NULL; } +int +dirvote_create_microdescriptor(char *out, size_t outlen, + const routerinfo_t *ri) +{ + char *key = NULL, *summary = NULL, *family = NULL; + size_t keylen; + int result = -1; + char *start = out, *end = out+outlen; + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + result = out - start; + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_CM_MICRODESC 7 +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +int +dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, + const char *microdesc, + size_t microdescriptor_len) +{ + char d[DIGEST256_LEN]; + char d64[BASE64_DIGEST256_LEN]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_CM_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (crypto_digest256(d, microdesc, microdescriptor_len, DIGEST_SHA256)<0) + return -1; + if (digest256_to_base64(d64, d)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + diff --git a/src/or/or.h b/src/or/or.h index d9e883f3e8..e7e3869793 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3823,6 +3823,12 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); +int dirvote_create_microdescriptor(char *out, + size_t outlen, const routerinfo_t *ri); +int dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, + const char *microdesc, + size_t microdescriptor_len); + #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, networkstatus_t *v3_ns); From a7ba02f3f1dc267be81bf36f16bc1e69b34af050 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Aug 2009 11:39:22 -0400 Subject: [PATCH 25/81] Add ability to parse one or more m line from a vote. --- src/or/or.h | 14 ++++++++++++++ src/or/routerparse.c | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/or/or.h b/src/or/or.h index e7e3869793..2c795421db 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1557,6 +1557,14 @@ typedef struct routerstatus_t { } routerstatus_t; +/**DOCDOC*/ +typedef struct microdescriptor_t { + crypto_pk_env_t *onion_pkey; + smartlist_t *family; + char *exitsummary; /**< exit policy summary - + * XXX weasel: this probably should not stay a string. */ +} microdescriptor_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1599,6 +1607,11 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +typedef struct vote_microdesc_hash_t { + struct vote_microdesc_hash_t *next; + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1607,6 +1620,7 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; /** Information about a single voter in a vote or a consensus. */ diff --git a/src/or/routerparse.c b/src/or/routerparse.c index bfcf20b09c..dd0d32ef63 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -55,6 +55,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -321,6 +322,7 @@ static token_rule_t rtrstatus_token_table[] = { T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -2050,6 +2052,18 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; From a8e92ba8fd65e12dbec265ccfcf0c89ac61847f2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 1 Sep 2009 02:00:43 -0400 Subject: [PATCH 26/81] Add a function to get the most frequent member of a list. --- src/common/container.c | 74 ++++++++++++++++++++++++++++++++++++++++++ src/common/container.h | 9 +++++ 2 files changed, 83 insertions(+) diff --git a/src/common/container.c b/src/common/container.c index 12ac2527e9..4fb94d3f7c 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -459,6 +459,42 @@ smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)) (int (*)(const void *,const void*))compare); } +/** Given a smartlist sl sorted with the function compare, + * return the most frequent member in the list. Break ties in favor of + * later elements. If the list is empty, return NULL. + */ +void * +smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)) +{ + const void *most_frequent = NULL; + int most_frequent_count = 0; + + const void **cur = NULL; + int i, count=0; + + if (!sl->num_used) + return NULL; + for (i = 0; i < sl->num_used; ++i) { + const void *item = sl->list[i]; + if (cur && 0 == compare(cur, &item)) { + ++count; + } else { + if (cur && count >= most_frequent_count) { + most_frequent = *cur; + most_frequent_count = count; + } + cur = &item; + count = 1; + } + } + if (cur && count >= most_frequent_count) { + most_frequent = *cur; + most_frequent_count = count; + } + return (void*)most_frequent; +} + /** Given a sorted smartlist sl and the comparison function used to * sort it, remove all duplicate members. If free_fn is provided, calls * free_fn on each duplicate. Otherwise, just removes them. Preserves order. @@ -550,6 +586,13 @@ smartlist_sort_strings(smartlist_t *sl) smartlist_sort(sl, _compare_string_ptrs); } +/** Return the most frequent string in the sorted list sl */ +char * +smartlist_get_most_frequent_string(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_string_ptrs); +} + /** Remove duplicate strings from a sorted list, and free them with tor_free(). */ void @@ -681,6 +724,37 @@ smartlist_uniq_digests(smartlist_t *sl) smartlist_uniq(sl, _compare_digests, _tor_free); } +/** Helper: compare two DIGEST256_LEN digests. */ +static int +_compare_digests256(const void **_a, const void **_b) +{ + return memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN); +} + +/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */ +void +smartlist_sort_digests256(smartlist_t *sl) +{ + smartlist_sort(sl, _compare_digests256); +} + +/** Return the most frequent member of the sorted list of DIGEST256_LEN + * digests in sl */ +char * +smartlist_get_most_frequent_digest256(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_digests256); +} + +/** Remove duplicate 256-bit digests from a sorted list, and free them with + * tor_free(). + */ +void +smartlist_uniq_digests256(smartlist_t *sl) +{ + smartlist_uniq(sl, _compare_digests256, _tor_free); +} + /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called maptype. The key part of each * entry is declared using the C declaration keydecl. All functions diff --git a/src/common/container.h b/src/common/container.h index 4495a7a273..41c0c68705 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -93,13 +93,22 @@ void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); void smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)); +void *smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)); void smartlist_uniq(smartlist_t *sl, int (*compare)(const void **a, const void **b), void (*free_fn)(void *elt)); + void smartlist_sort_strings(smartlist_t *sl); void smartlist_sort_digests(smartlist_t *sl); +void smartlist_sort_digests256(smartlist_t *sl); + +char *smartlist_get_most_frequent_string(smartlist_t *sl); +char *smartlist_get_most_frequent_digest256(smartlist_t *sl); + void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); +void smartlist_uniq_digests256(smartlist_t *sl); void *smartlist_bsearch(smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) ATTR_PURE; From e1ddee8bbe724e934fe9a4cb2d290719a7d6105c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 24 Aug 2009 12:51:33 -0400 Subject: [PATCH 27/81] Code to generate, store, and parse microdescriptors and consensuses. The consensus documents are not signed properly, not served, and not exchanged yet. --- src/common/util.c | 7 ++ src/common/util.h | 1 + src/or/Makefile.am | 1 + src/or/dirserv.c | 35 +++++- src/or/dirvote.c | 200 +++++++++++++++++++++--------- src/or/microdesc.c | 267 +++++++++++++++++++++++++++++++++++++++++ src/or/networkstatus.c | 6 + src/or/or.h | 73 +++++++++-- src/or/routerparse.c | 206 ++++++++++++++++++++++++++++--- src/test/test_dir.c | 9 +- 10 files changed, 712 insertions(+), 93 deletions(-) create mode 100644 src/or/microdesc.c diff --git a/src/common/util.c b/src/common/util.c index d05c308fe8..139c1aaad8 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -684,6 +684,13 @@ tor_digest_is_zero(const char *digest) return tor_mem_is_zero(digest, DIGEST_LEN); } +/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ +int +tor_digest256_is_zero(const char *digest) +{ + return tor_mem_is_zero(digest, DIGEST256_LEN); +} + /* Helper: common code to check whether the result of a strtol or strtoul or * strtoll is correct. */ #define CHECK_STRTOX_RESULT() \ diff --git a/src/common/util.h b/src/common/util.h index 28ea8a0488..85234f5157 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -195,6 +195,7 @@ const char *find_whitespace(const char *s) ATTR_PURE; const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE; int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE; int tor_digest_is_zero(const char *digest) ATTR_PURE; +int tor_digest256_is_zero(const char *digest) ATTR_PURE; char *esc_for_log(const char *string) ATTR_MALLOC; const char *escaped(const char *string); struct smartlist_t; diff --git a/src/or/Makefile.am b/src/or/Makefile.am index 097e3e24de..3dc1889a90 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -20,6 +20,7 @@ libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ diff --git a/src/or/dirserv.c b/src/or/dirserv.c index f12ef2f3d5..e7a58edfc1 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1901,10 +1901,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1918,7 +1919,7 @@ routerstatus_format_entry(char *buf, size_t buf_len, * this here, instead of in the caller. Then we could use the * networkstatus_type_t values, with an additional control port value * added -MP */ - if (format == NS_V3_CONSENSUS) + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -2434,6 +2435,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); @@ -2482,11 +2484,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; @@ -2501,9 +2505,30 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); diff --git a/src/or/dirvote.c b/src/or/dirvote.c index f0f92d2f60..d200344ef8 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -24,14 +24,20 @@ static int dirvote_publish_consensus(void); static char *make_consensus_method_list(int low, int high, const char *sep); /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 7 +#define MAX_SUPPORTED_CONSENSUS_METHOD 8 #define MIN_METHOD_FOR_PARAMS 7 +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 + /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * v3_ns, signed with our v3 signing key private_signing_key. * For v3 authorities. */ @@ -89,7 +95,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -158,15 +164,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -189,7 +205,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, @@ -294,34 +310,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings lst, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if a and b are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -363,7 +353,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -399,6 +390,26 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH(votes, vote_routerstatus_t *, rs, { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + }); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } @@ -615,16 +626,20 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); if (!smartlist_len(votes)) { @@ -955,6 +970,7 @@ networkstatus_compute_consensus(smartlist_t *votes, int n_listing = 0; int i; char buf[256]; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -1021,7 +1037,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -1182,9 +1200,19 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Okay!! Now we can write the descriptor... */ /* First line goes into "buf". */ routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, - NS_V3_CONSENSUS); + rs_format); smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + const size_t mlen = BASE64_DIGEST256_LEN+5; + digest256_to_base64(m, microdesc_digest); + cp = tor_malloc(mlen); + tor_snprintf(cp, mlen, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1206,7 +1234,7 @@ networkstatus_compute_consensus(smartlist_t *votes, }; /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { + if (rs_out.has_exitsummary && flavor == FLAV_NS) { char buf[MAX_POLICY_LINE_LEN+1]; int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); if (r<0) { @@ -2090,7 +2118,8 @@ dirvote_compute_consensus(void) consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + FLAV_NS); } if (!consensus_body) { log_warn(LD_DIR, "Couldn't generate a consensus at all!"); @@ -2389,14 +2418,21 @@ dirvote_get_vote(const char *fp, int flags) return NULL; } -int -dirvote_create_microdescriptor(char *out, size_t outlen, - const routerinfo_t *ri) +/** Construct and return a new microdescriptor from a routerinfo ri. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) { + microdesc_t *result = NULL; char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; size_t keylen; - int result = -1; - char *start = out, *end = out+outlen; + char *out = buf, *end = buf+sizeof(buf); if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) goto done; @@ -2417,7 +2453,19 @@ dirvote_create_microdescriptor(char *out, size_t outlen, goto done; out += strlen(out); } - result = out - start; + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } done: tor_free(key); @@ -2426,28 +2474,23 @@ dirvote_create_microdescriptor(char *out, size_t outlen, return result; } -/** Lowest consensus method that generates microdescriptors */ -#define MIN_CM_MICRODESC 7 /** Cached space-separated string to hold */ static char *microdesc_consensus_methods = NULL; +/** DOCDOC */ int -dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, - const char *microdesc, - size_t microdescriptor_len) +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) { - char d[DIGEST256_LEN]; char d64[BASE64_DIGEST256_LEN]; if (!microdesc_consensus_methods) { microdesc_consensus_methods = - make_consensus_method_list(MIN_CM_MICRODESC, + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD, ","); tor_assert(microdesc_consensus_methods); } - if (crypto_digest256(d, microdesc, microdescriptor_len, DIGEST_SHA256)<0) - return -1; - if (digest256_to_base64(d64, d)<0) + if (digest256_to_base64(d64, md->digest)<0) return -1; if (tor_snprintf(out, out_len, "m %s sha256=%s\n", @@ -2457,3 +2500,44 @@ dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, return strlen(out); } +/** DOCDOC */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && !memcmp(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, " sha256="); + if (!cp) + return -1; + cp += strlen(" sha256="); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 0000000000..0128fbbab2 --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,267 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" + +/** DOCDOC everything here. */ + +#define MICRODESC_IN_CONSENSUS 1 +#define MICRODESC_IN_VOTE 2 + +struct microdesc_cache_t { + HT_HEAD(microdesc_map, microdesc_t) map; + + char *cache_fname; + char *journal_fname; + tor_mmap_t *cache_content; + size_t journal_len; +}; + +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return !memcmp(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + _tor_malloc, _tor_realloc, _tor_free); + +static int +dump_microdescriptor(FILE *f, microdesc_t *md) +{ + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + format_iso_time(buf, md->last_listed); + fprintf(f, "@last-listed %s\n", buf); + } + + md->off = (off_t) ftell(f); + fwrite(md->body, 1, md->bodylen, f); + return 0; +} + +static microdesc_cache_t *the_microdesc_cache = NULL; + +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/* Returns list of added microdesc_t. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* Returns list of added microdesc_t. Frees any not added. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!f) + log_warn(LD_DIR, "Couldn't append to journal in %s", + cache->journal_fname); + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + continue; + } + + /* Okay, it's a new one. */ + if (f) { + dump_microdescriptor(f, md); + md->saved_location = SAVED_IN_JOURNAL; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + } SMARTLIST_FOREACH_END(md); + + finish_writing_to_file(open_file); /*XXX Check me.*/ + return added; +} + +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(*entry); + } + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } +} + +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + total += smartlist_len(added); + smartlist_free(added); + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + total += smartlist_len(added); + smartlist_free(added); + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + return 0; +} + +int +microdesc_cache_rebuild(microdesc_cache_t *cache) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + + f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + if (md->no_save) + continue; + + dump_microdescriptor(f, md); + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + + smartlist_add(wrote, md); + } + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + cache->cache_content = tor_mmap_file(cache->cache_fname); + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + if (md->no_save) + continue; + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(!memcmp(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(wrote); + + smartlist_free(wrote); + + return 0; +} + +void +microdesc_free(microdesc_t *md) +{ + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 5d1f8b24a3..752cb42124 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -242,8 +242,14 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } diff --git a/src/or/or.h b/src/or/or.h index 2c795421db..6a7aec6738 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -89,6 +89,7 @@ #include "torgzip.h" #include "address.h" #include "compat_libevent.h" +#include "ht.h" /* These signals are defined to help control_signal_act work. */ @@ -1558,12 +1559,29 @@ typedef struct routerstatus_t { } routerstatus_t; /**DOCDOC*/ -typedef struct microdescriptor_t { +typedef struct microdesc_t { + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + time_t last_listed; + saved_location_t saved_location : 3; + unsigned int no_save : 1; + off_t off; + + /* The string containing the microdesc. */ + + char *body; + size_t bodylen; + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + crypto_pk_env_t *onion_pkey; smartlist_t *family; char *exitsummary; /**< exit policy summary - - * XXX weasel: this probably should not stay a string. */ -} microdescriptor_t; + * XXX this probably should not stay a string. */ +} microdesc_t; /** How many times will we try to download a router's descriptor before giving * up? */ @@ -3748,7 +3766,8 @@ int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, int compressed); typedef enum { - NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT + NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT, + NS_V3_CONSENSUS_MICRODESC } routerstatus_format_type_t; int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *platform, @@ -3784,13 +3803,20 @@ int dirserv_read_measured_bandwidths(const char *from_file, void dirvote_free_all(void); +/** DOCDOC */ +typedef enum { + FLAV_NS, + FLAV_MICRODESC, +} consensus_flavor_t; + /* vote manipulation */ char *networkstatus_compute_consensus(smartlist_t *votes, int total_authorities, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char **msg_out); @@ -3837,11 +3863,12 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); -int dirvote_create_microdescriptor(char *out, - size_t outlen, const routerinfo_t *ri); -int dirvote_format_microdescriptor_vote_line(char *out, size_t out_len, - const char *microdesc, - size_t microdescriptor_len); +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +int dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method); #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, @@ -4051,6 +4078,25 @@ void do_hash_password(void); int tor_init(int argc, char **argv); #endif +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); + /********************************* networkstatus.c *********************/ /** How old do we allow a v2 network-status to get before removing it @@ -4927,7 +4973,8 @@ int router_get_router_hash(const char *s, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_runningrouters_hash(const char *s, char *digest); int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, @@ -4971,6 +5018,10 @@ networkstatus_t *networkstatus_parse_vote_from_string(const char *s, ns_detached_signatures_t *networkstatus_parse_detached_signatures( const char *s, const char *eos); +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + authority_cert_t *authority_cert_parse_from_string(const char *s, const char **end_of_string); int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, diff --git a/src/or/routerparse.c b/src/or/routerparse.c index dd0d32ef63..277c7c6c91 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -111,6 +111,7 @@ typedef enum { K_LEGACY_DIR_KEY, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -496,6 +497,14 @@ static token_rule_t networkstatus_detached_signature_token_table[] = { END_OF_TABLE }; +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), + END_OF_TABLE +}; + #undef T /* static function prototypes */ @@ -505,7 +514,8 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, char *digest, const char *start_str, const char *end_str, - char end_char); + char end_char, + digest_algorithm_t alg); static void token_free(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, @@ -586,7 +596,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the first router in @@ -596,7 +607,8 @@ int router_get_router_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the running-routers @@ -606,7 +618,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the network-status @@ -616,17 +629,19 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); } /** Set digest to the SHA-1 digest of the hash of the network-status * string in s. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s,digest, "network-status-version","\ndirectory-signature", - ' '); + ' ', alg); } /** Set digest to the SHA-1 digest of the hash of the extrainfo @@ -634,7 +649,8 @@ router_get_networkstatus_v3_hash(const char *s, char *digest) int router_get_extrainfo_hash(const char *s, char *digest) { - return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n'); + return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n', + DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -643,6 +659,8 @@ router_get_extrainfo_hash(const char *s, char *digest) * surround it with -----BEGIN/END----- pairs, and write it to the * buf_len-byte buffer at buf. Return 0 on success, -1 on * failure. + * + * DOCDOC alg */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, @@ -1691,7 +1709,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -2298,7 +2316,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hash(s, ns_digest, DIGEST_SHA1)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -3410,7 +3428,7 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of s taken from the first +/** Compute the digest of the substring of s taken from the first * occurrence of start_str through the first instance of c after the * first subsequent occurrence of end_str; store the 20-byte result in * digest; return 0 on success. @@ -3420,7 +3438,8 @@ find_all_exitpolicy(smartlist_t *s) static int router_get_hash_impl(const char *s, char *digest, const char *start_str, - const char *end_str, char end_c) + const char *end_str, char end_c, + digest_algorithm_t alg) { char *start, *end; start = strstr(s, start_str); @@ -3446,14 +3465,169 @@ router_get_hash_impl(const char *s, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); - return -1; + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } } return 0; } +/** DOCDOC Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/**DOCDOC*/ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;in_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + if (md) + microdesc_free(md); + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string platform, * and compare it to the version in cutoff. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3712,7 +3886,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } diff --git a/src/test/test_dir.c b/src/test/test_dir.c index a10493e7fa..8e566e2eec 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -858,7 +858,8 @@ test_dir_v3_networkstatus(void) cert3->identity_key, sign_skey_3, "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); + sign_skey_leg1, + FLAV_NS); test_assert(consensus_text); con = networkstatus_parse_vote_from_string(consensus_text, NULL, NS_TYPE_CONSENSUS); @@ -966,11 +967,13 @@ test_dir_v3_networkstatus(void) smartlist_shuffle(votes); consensus_text2 = networkstatus_compute_consensus(votes, 3, cert2->identity_key, - sign_skey_2, NULL,NULL); + sign_skey_2, NULL,NULL, + FLAV_NS); smartlist_shuffle(votes); consensus_text3 = networkstatus_compute_consensus(votes, 3, cert1->identity_key, - sign_skey_1, NULL,NULL); + sign_skey_1, NULL,NULL, + FLAV_NS); test_assert(consensus_text2); test_assert(consensus_text3); con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, From 3b2fc659a8ef83feedadcda32de49db06b80af10 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Sep 2009 17:01:01 -0400 Subject: [PATCH 28/81] Refactor consensus signature storage for multiple digests and flavors. This patch introduces a new type called document_signature_t to represent the signature of a consensus document. Now, each consensus document can have up to one document signature per voter per digest algorithm. Also, each detached-signatures document can have up to one signature per . --- src/common/crypto.c | 33 ++++++ src/common/crypto.h | 21 +++- src/or/directory.c | 10 +- src/or/dirserv.c | 7 +- src/or/dirvote.c | 213 ++++++++++++++++++++++++-------------- src/or/networkstatus.c | 159 +++++++++++++++++++---------- src/or/or.h | 78 +++++++++----- src/or/routerlist.c | 74 +++++++------- src/or/routerparse.c | 226 +++++++++++++++++++++++++++-------------- src/test/test_dir.c | 74 ++++++++------ 10 files changed, 589 insertions(+), 306 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index 21c8aed2d4..ac0e628c48 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1448,6 +1448,39 @@ crypto_digest256(char *digest, const char *m, size_t len, return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL); } +/** Set the digests_t in ds_out to contain every digest on the + * len bytes in m that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_digest_all(digests_t *ds_out, const char *m, size_t len) +{ + digest_algorithm_t i; + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) { + if (crypto_digest256(ds_out->d[i], m, len, i) < 0) + return -1; + } + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + default: + tor_fragile_assert(); + return "??unknown_digest??"; + } +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { union { diff --git a/src/common/crypto.h b/src/common/crypto.h index 63ea96d056..ed8468046f 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -58,9 +58,22 @@ #define HEX_DIGEST256_LEN 64 typedef enum { - DIGEST_SHA1, - DIGEST_SHA256, + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, } digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +/** A set of all the digests we know how to compute, taken on a single + * string. Any digests that are shorter than 256 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 12 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} digests_t; typedef struct crypto_pk_env_t crypto_pk_env_t; typedef struct crypto_cipher_env_t crypto_cipher_env_t; @@ -158,10 +171,12 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 */ +/* SHA-1 and other digests. */ int crypto_digest(char *digest, const char *m, size_t len); int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm); +int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); crypto_digest_env_t *crypto_new_digest_env(void); crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); diff --git a/src/or/directory.c b/src/or/directory.c index 5fe2de4eee..19ef6351c2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2330,7 +2330,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) @@ -2341,18 +2341,18 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && !memcmp(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index e7a58edfc1..326a801c41 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -2597,12 +2597,17 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); + { + document_signature_t *sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, identity_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, signing_key_digest, DIGEST_LEN); + } voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; voter->or_port = options->ORPort; voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { diff --git a/src/or/dirvote.c b/src/or/dirvote.c index d200344ef8..1bdf4cc403 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -251,6 +251,19 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** DOCDOC */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -782,8 +795,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -797,7 +809,7 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, @@ -1347,84 +1359,123 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { + puts("A"); *msg_out = "Valid-After times do not match " "when adding detached signatures to consensus"; return -1; } if (target->fresh_until != sigs->fresh_until) { + puts("B"); *msg_out = "Fresh-until times do not match " "when adding detached signatures to consensus"; return -1; } if (target->valid_until != sigs->valid_until) { + puts("C"); *msg_out = "Valid-until times do not match " "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + puts("D"); + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; - } - - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } - - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + puts("D"); + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { + ++n_matches; + } else { + printf("F %d\n", alg); + printf("%s\n", hex_str(target->digests.d[alg], DIGEST256_LEN)); + printf("%s\n", hex_str(digests->d[alg], DIGEST256_LEN)); + *msg_out = "Mismatched digest."; + return -1; } } + } + if (!n_matches) { + puts("G"); + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; - } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } + + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); return r; } @@ -1441,6 +1492,8 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) tor_assert(consensus); tor_assert(consensus->type == NS_TYPE_CONSENSUS); + tor_assert(consensus->flavor == FLAV_NS); + elements = smartlist_create(); { @@ -1448,7 +1501,7 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); + base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); format_iso_time(va_buf, consensus->valid_after); format_iso_time(fu_buf, consensus->fresh_until); format_iso_time(vu_buf, consensus->valid_until); @@ -1461,26 +1514,26 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { char sk[HEX_DIGEST_LEN+1]; char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + if (!sig->signature || sig->bad_signature || sig->alg != DIGEST_SHA1) continue; ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); tor_snprintf(buf, sizeof(buf), "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", id, sk); smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); smartlist_add(elements, tor_strdup(buf)); - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); result = smartlist_join_strings(elements, "", 0, NULL); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); if (!n_sigs) @@ -1493,13 +1546,15 @@ void ns_detached_signatures_free(ns_detached_signatures_t *s) { if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1936,7 +1991,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -2218,8 +2279,12 @@ dirvote_add_signatures_to_pending_consensus( goto err; } - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); + { + smartlist_t *sig_list = strmap_get(sigs->signatures, + networkstatus_get_flavor_name(pending_consensus->flavor)); + log_info(LD_DIR, "Have %d signatures for adding to consensus.", + sig_list ? smartlist_len(sig_list) : 0); + } r = networkstatus_add_detached_signatures(pending_consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); @@ -2406,12 +2471,12 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 752cb42124..616726525b 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -279,7 +279,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in ns. */ +/** Free all storage held in sig */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of sig */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in ns. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -301,14 +319,17 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } if (ns->cert) @@ -347,34 +368,38 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on voter is correctly signed by - * the signing key of cert. Return -1 if cert doesn't match the +/** Check whether the signature sig is correctly signed with the + * signing key in cert. Return -1 if cert doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * voter, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (memcmp(voter->signing_key_digest, d, DIGEST_LEN)) + if (memcmp(sig->signing_key_digest, key_digest, DIGEST_LEN) || + memcmp(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, - voter->signature, - voter->signature_len) != DIGEST_LEN || - memcmp(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + memcmp(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -407,37 +432,52 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int missing_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(!memcmp(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++n_unknown; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; + else if (missing_key_here) + ++n_missing_key; else ++n_no_signature; - }); + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -1434,8 +1474,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } if (current_consensus && - !memcmp(c->networkstatus_digest, current_consensus->networkstatus_digest, - DIGEST_LEN)) { + !memcmp(&c->digests, ¤t_consensus->digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ log_info(LD_DIR, "Got a consensus we already have"); goto done; @@ -1669,10 +1708,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1681,8 +1718,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1930,6 +1967,22 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, return default_val; } +/** Return the name of the consensus flavor flav as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} + /** If question is a string beginning with "ns/" in a format the * control interface expects for a GETINFO question, set *answer to a * newly-allocated string containing networkstatus lines for the appropriate diff --git a/src/or/or.h b/src/or/or.h index 6a7aec6738..e2050ecae4 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1641,29 +1641,42 @@ typedef struct vote_routerstatus_t { vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of signature */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of signature */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1673,10 +1686,17 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** DOCDOC */ +typedef enum { + FLAV_NS, + FLAV_MICRODESC, +} consensus_flavor_t; + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1715,8 +1735,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1728,14 +1748,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -3803,12 +3824,6 @@ int dirserv_read_measured_bandwidths(const char *from_file, void dirvote_free_all(void); -/** DOCDOC */ -typedef enum { - FLAV_NS, - FLAV_MICRODESC, -} consensus_flavor_t; - /* vote manipulation */ char *networkstatus_compute_consensus(smartlist_t *votes, int total_authorities, @@ -3869,6 +3884,9 @@ int dirvote_format_microdesc_vote_line(char *out, size_t out_len, int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, int method); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, @@ -4134,9 +4152,9 @@ networkstatus_voter_info_t *networkstatus_get_voter_by_id( const char *identity); int networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); char *networkstatus_get_cache_filename(const char *identity_digest); int router_set_networkstatus_v2(const char *s, time_t arrived_at, v2_networkstatus_source_t source, @@ -4189,6 +4207,9 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int32_t default_val); int getinfo_helper_networkstatus(control_connection_t *conn, const char *question, char **answer); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); /********************************* ntmain.c ***************************/ @@ -4975,6 +4996,7 @@ int router_get_runningrouters_hash(const char *s, char *digest); int router_get_networkstatus_v2_hash(const char *s, char *digest); int router_get_networkstatus_v3_hash(const char *s, char *digest, digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 0a32f78a69..5ae40ddd67 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -448,17 +448,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -469,37 +470,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (!ftime_definitely_after(now, cert->expires)) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_notice(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 277c7c6c91..285a550acd 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -516,6 +516,9 @@ static int router_get_hash_impl(const char *s, char *digest, const char *start_str, const char *end_str, char end_char, digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, digests_t *digests, + const char *start_str, const char *end_str, + char end_char); static void token_free(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, @@ -633,6 +636,16 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) DIGEST_SHA1); } +/** DOCDOC */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,digests, + "network-status-version", + "\ndirectory-signature", + ' '); +} + /** Set digest to the SHA-1 digest of the hash of the network-status * string in s. Return 0 on success, -1 on failure. */ int @@ -640,7 +653,8 @@ router_get_networkstatus_v3_hash(const char *s, char *digest, digest_algorithm_t alg) { return router_get_hash_impl(s,digest, - "network-status-version","\ndirectory-signature", + "network-status-version", + "\ndirectory-signature", ' ', alg); } @@ -2304,7 +2318,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; + digests_t ns_digests; const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; @@ -2316,7 +2330,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest, DIGEST_SHA1)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2332,7 +2346,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2486,8 +2500,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2622,10 +2637,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, - { + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; @@ -2650,11 +2665,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = DIGEST_SHA1; if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), tok->args[1], HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared digest %s in " "network-status vote.", escaped(tok->args[1])); + tor_free(sig); goto err; } @@ -2663,32 +2682,42 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, DIGEST_LEN, + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, tok, ns->cert->signing_key, 0, - "network-status vote")) + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; - } else { - if (tok->object_size >= INT_MAX) - goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } + sig->good_signature = 1; + } else { + if (tok->object_size >= INT_MAX) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); @@ -2714,10 +2743,14 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { @@ -2747,10 +2780,19 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + const char *flavor = "ns"; + digests_t *digests; + smartlist_t *sig_list; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); + digests = tor_malloc_zero(sizeof(digests_t)); + sig_list = smartlist_create(); + strmap_set(sigs->digests, flavor, digests); + strmap_set(sigs->signatures, flavor, sig_list); if (!eos) eos = s + strlen(s); @@ -2768,7 +2810,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) "networkstatus signatures"); goto err; } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, + if (base16_decode(digests->d[DIGEST_SHA1], DIGEST_LEN, tok->args[0], strlen(tok->args[0])) < 0) { log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " "networkstatus signatures"); @@ -2793,50 +2835,51 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + document_signature_t *sig; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; - tor_assert(tok->n_args >= 2); + tok = _tok; + if (tok->tp != K_DIRECTORY_SIGNATURE) + continue; + tor_assert(tok->n_args >= 2); - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + tok->args[0], HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(tok->args[0])); + goto err; + } + if (strlen(tok->args[1]) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + tok->args[1], HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared digest %s in " + "network-status vote.", escaped(tok->args[1])); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = DIGEST_SHA1; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; - smartlist_add(sigs->signatures, voter); - }); + smartlist_add(sig_list, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: @@ -3428,18 +3471,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the digest of the substring of s taken from the first - * occurrence of start_str through the first instance of c after the - * first subsequent occurrence of end_str; store the 20-byte result in - * digest; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, char *digest, - const char *start_str, - const char *end_str, char end_c, - digest_algorithm_t alg) +router_get_hash_impl_helper(const char *s, + const char *start_str, + const char *end_str, char end_c, + const char **start_out, const char **end_out) { char *start, *end; start = strstr(s, start_str); @@ -3465,6 +3501,28 @@ router_get_hash_impl(const char *s, char *digest, } ++end; + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of s taken from the first + * occurrence of start_str through the first instance of c after the + * first subsequent occurrence of end_str; store the 20-byte result in + * digest; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,start_str,end_str,end_c,&start,&end)<0) + return -1; + if (alg == DIGEST_SHA1) { if (crypto_digest(digest, start, end-start)) { log_warn(LD_BUG,"couldn't compute digest"); @@ -3480,6 +3538,24 @@ router_get_hash_impl(const char *s, char *digest, return 0; } +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,start_str,end_str,end_c,&start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); + return -1; + } + + return 0; +} + /** DOCDOC Assuming that s starts with a microdesc, return the start of the * *NEXT* one. */ static const char * diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 8e566e2eec..b1fac683cc 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -571,6 +571,7 @@ test_dir_v3_networkstatus(void) time_t now = time(NULL); networkstatus_voter_info_t *voter; + document_signature_t *sig; networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; vote_routerstatus_t *vrs; routerstatus_t *rs; @@ -946,20 +947,25 @@ test_dir_v3_networkstatus(void) /* Check signatures. the first voter is a pseudo-entry with a legacy key. * The second one hasn't signed. The fourth one has signed: validate it. */ voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); + test_eq(smartlist_len(voter->sigs), 0); +#if 0 + sig = smartlist_get(voter->sigs, 1); + test_assert(!sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); +#endif voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); + test_eq(smartlist_len(voter->sigs), 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); + + test_assert(!networkstatus_check_document_signature(con, sig, cert3)); + test_assert(sig->signature); + test_assert(sig->good_signature); + test_assert(!sig->bad_signature); { const char *msg=NULL; @@ -984,10 +990,8 @@ test_dir_v3_networkstatus(void) test_assert(con3); /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); + test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); + test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); /* Extract a detached signature from con3. */ detached_text1 = networkstatus_get_detached_signatures(con3); @@ -1000,12 +1004,20 @@ test_dir_v3_networkstatus(void) test_eq(dsig1->valid_after, con3->valid_after); test_eq(dsig1->fresh_until, con3->fresh_until); test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); + { + digests_t *dsig_digests = strmap_get(dsig1->digests, "ns"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], + DIGEST_LEN); + } + { + smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + } /* Try adding it to con2. */ detached_text2 = networkstatus_get_detached_signatures(con2); @@ -1023,7 +1035,8 @@ test_dir_v3_networkstatus(void) printf("%s\n", hd); }); */ - test_eq(2, smartlist_len(dsig2->signatures)); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); /* Try adding to con2 twice; verify that nothing changes. */ test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); @@ -1031,13 +1044,14 @@ test_dir_v3_networkstatus(void) /* Add to con. */ test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - + voter = smartlist_get(con->voters, 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert2)); + voter = smartlist_get(con->voters, 2); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert1)); } done: From 5576a3a094de49246e501bb24d1b3dc3a5d610b9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Sep 2009 12:34:44 -0400 Subject: [PATCH 29/81] Parse detached signature documents with multiple flavors and algorithms. --- src/common/crypto.c | 12 ++++ src/common/crypto.h | 1 + src/or/routerparse.c | 160 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 146 insertions(+), 27 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index ac0e628c48..f335f99aba 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1481,6 +1481,18 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg) } } +/** DOCDOC */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else + return -1; +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { union { diff --git a/src/common/crypto.h b/src/common/crypto.h index ed8468046f..c0a4526255 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -177,6 +177,7 @@ int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm); int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_env_t *crypto_new_digest_env(void); crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 285a550acd..21fe5f7d23 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -106,6 +106,8 @@ typedef enum { K_PARAMS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, @@ -490,10 +492,12 @@ static token_rule_t networkstatus_vote_footer_token_table[] = { /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), END_OF_TABLE }; @@ -2771,6 +2775,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** DOCDOC */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** DOCDOC */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between s and * eos and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2780,19 +2809,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; - const char *flavor = "ns"; digests_t *digests; - smartlist_t *sig_list; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); sigs->digests = strmap_new(); sigs->signatures = strmap_new(); - digests = tor_malloc_zero(sizeof(digests_t)); - sig_list = smartlist_create(); - strmap_set(sigs->digests, flavor, digests); - strmap_set(sigs->signatures, flavor, sig_list); if (!eos) eos = s + strlen(s); @@ -2804,18 +2827,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(digests->d[DIGEST_SHA1], DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2836,14 +2898,43 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) } SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + char id_digest[DIGEST_LEN]; char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; document_signature_t *sig; + int is_duplicate; tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) + if (tok->tp == K_DIRECTORY_SIGNATURE) { + tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { continue; - tor_assert(tok->n_args >= 2); + } + + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; + } + alg = (digest_algorithm_t) a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2852,21 +2943,36 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (strlen(tok->args[1]) != HEX_DIGEST_LEN || base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { + sk_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + "network-status vote.", escaped(sk_hexdigest)); goto err; } + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + !memcmp(id_digest, s->identity_digest, DIGEST_LEN) && + !memcmp(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } + }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + sig = tor_malloc_zero(sizeof(document_signature_t)); sig->alg = DIGEST_SHA1; memcpy(sig->identity_digest, id_digest, DIGEST_LEN); @@ -2878,7 +2984,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) sig->signature = tor_memdup(tok->object_body, tok->object_size); sig->signature_len = (int) tok->object_size; - smartlist_add(sig_list, sig); + smartlist_add(siglist, sig); } SMARTLIST_FOREACH_END(_tok); goto done; From d9c71816b15af4325fb8ab9befa8600185d9aa90 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 22 Sep 2009 16:52:51 -0400 Subject: [PATCH 30/81] Generate all the flavors of consensuses when building consensuses. --- src/or/directory.c | 2 +- src/or/dirvote.c | 350 ++++++++++++++++++++++++++++++--------------- src/or/or.h | 9 +- 3 files changed, 241 insertions(+), 120 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index 19ef6351c2..0feddcbae4 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2618,7 +2618,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 1bdf4cc403..e3611025e9 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,15 +11,21 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +typedef struct pending_consensus_t pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); static char *make_consensus_method_list(int low, int high, const char *sep); @@ -1480,40 +1486,17 @@ networkstatus_add_detached_signatures(networkstatus_t *target, return r; } -/** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on consensus. */ -char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +/** DOCDOC */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus) { smartlist_t *elements; char buf[4096]; char *result = NULL; int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); - - tor_assert(consensus->flavor == FLAV_NS); elements = smartlist_create(); - { - char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], - vu_buf[ISO_TIME_LEN+1]; - char d[HEX_DIGEST_LEN+1]; - - base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); - - tor_snprintf(buf, sizeof(buf), - "consensus-digest %s\n" - "valid-after %s\n" - "fresh-until %s\n" - "valid-until %s\n", d, va_buf, fu_buf, vu_buf); - smartlist_add(elements, tor_strdup(buf)); - } - SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { char sk[HEX_DIGEST_LEN+1]; @@ -1541,6 +1524,53 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) return result; } +/** Return a newly allocated string holding the detached-signatures document + * corresponding to the signatures on consensus. */ +char * +networkstatus_get_detached_signatures(networkstatus_t *consensus) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + tor_assert(consensus); + tor_assert(consensus->type == NS_TYPE_CONSENSUS); + + tor_assert(consensus->flavor == FLAV_NS); + + elements = smartlist_create(); + + { + char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], + vu_buf[ISO_TIME_LEN+1]; + char d[HEX_DIGEST_LEN+1]; + + base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus->valid_after); + format_iso_time(fu_buf, consensus->fresh_until); + format_iso_time(vu_buf, consensus->valid_until); + + tor_snprintf(buf, sizeof(buf), + "consensus-digest %s\n" + "valid-after %s\n" + "fresh-until %s\n" + "valid-until %s\n", d, va_buf, fu_buf, vu_buf); + smartlist_add(elements, tor_strdup(buf)); + } + + { + char *sigs = networkstatus_format_signatures(consensus); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + } + + result = smartlist_join_strings(elements, "", 0, NULL); + err: + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + return result; +} + /** Release all storage held in s. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) @@ -1744,7 +1774,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1781,14 +1811,22 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +/** DOCDOC*/ +struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +}; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1882,15 +1920,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** DOCDOC */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + if (pc->consensus) { + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1928,12 +1990,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -2121,7 +2179,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ int n_votes, n_voters; @@ -2129,6 +2187,10 @@ dirvote_compute_consensus(void) char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -2168,6 +2230,7 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); @@ -2176,40 +2239,59 @@ dirvote_compute_consensus(void) legacy_id_digest = legacy_dbuf; } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, - FLAV_NS); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; - } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } + } + + /* XXXX NMNM NM NM wrong. */ + signatures = + networkstatus_get_detached_signatures(pending[FLAV_NS].consensus); - signatures = networkstatus_get_detached_signatures(consensus); if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -2217,7 +2299,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2253,79 +2335,61 @@ dirvote_compute_consensus(void) } /** Helper: we just got the detached_signatures_body sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to pc * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; - } - { - smartlist_t *sig_list = strmap_get(sigs->signatures, - networkstatus_get_flavor_name(pending_consensus->flavor)); - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - sig_list ? smartlist_len(sig_list) : 0); + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; } else if (r == 0) { *msg_out = "Signatures ignored"; @@ -2333,6 +2397,58 @@ dirvote_add_signatures_to_pending_consensus( goto err; } + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors) { + r = -1; + goto err; + } + + /* Still not right XXXX NM NM*/ + if (pending_consensuses[FLAV_NS].consensus) { + char *new_detached = networkstatus_get_detached_signatures( + pending_consensuses[FLAV_NS].consensus); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; goto done; err: if (!*msg_out) @@ -2340,6 +2456,8 @@ dirvote_add_signatures_to_pending_consensus( done: if (sigs) ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2352,10 +2470,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2375,13 +2493,16 @@ static int dirvote_publish_consensus(void) { /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { + if (!pending_consensuses[FLAV_NS].consensus || + networkstatus_check_consensus_signature( + pending_consensuses[FLAV_NS].consensus, 1)<0) { log_warn(LD_DIR, "Not enough info to publish pending consensus"); return -1; } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) + /* XXXXXX NMNMNM */ + if (networkstatus_set_current_consensus( + pending_consensuses[FLAV_NS].body, 0)) log_warn(LD_DIR, "Error publishing consensus"); else log_notice(LD_DIR, "Consensus published."); @@ -2400,12 +2521,8 @@ dirvote_free_all(void) smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { /* now empty as a result of clear_pending_votes. */ smartlist_free(pending_consensus_signature_list); @@ -2419,9 +2536,10 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently diff --git a/src/or/or.h b/src/or/or.h index e2050ecae4..b8ccac9239 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1688,10 +1688,13 @@ typedef enum { /** DOCDOC */ typedef enum { - FLAV_NS, - FLAV_MICRODESC, + FLAV_NS = 0, + FLAV_MICRODESC = 1, } consensus_flavor_t; +/** DOCDOC */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { @@ -3863,7 +3866,7 @@ int dirvote_add_signatures(const char *detached_signatures_body, const char **msg_out); /* Item access */ -const char *dirvote_get_pending_consensus(void); +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); const char *dirvote_get_pending_detached_signatures(void); #define DGV_BY_ID 1 #define DGV_INCLUDE_PENDING 2 From 3471057486a8aef0be6e74b090a3173e0794c84b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 23 Sep 2009 03:02:32 -0400 Subject: [PATCH 31/81] Implement signatures for microdesc consensuses right. This means we need to handle the existence of multiple flavors of signature in a detached signatures document, generate them correctly, and so on. --- src/or/dirserv.c | 1 + src/or/dirvote.c | 216 +++++++++++++++++++++++++++++++++----------- src/or/or.h | 2 +- src/test/test_dir.c | 21 ++++- 4 files changed, 183 insertions(+), 57 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 326a801c41..17fff6110c 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1876,6 +1876,7 @@ version_from_platform(const char *platform) * The format argument has three possible values: * NS_V2 - Output an entry suitable for a V2 NS opinion document * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - DOCDOC * NS_V3_VOTE - Output a complete V3 NS vote * NS_CONTROL_PORT - Output a NS document for the control port */ diff --git a/src/or/dirvote.c b/src/or/dirvote.c index e3611025e9..d64e4d53a2 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,7 +11,14 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -typedef struct pending_consensus_t pending_consensus_t; +/** DOCDOC*/ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, @@ -148,7 +155,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, flags, params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); tor_free(params); tor_free(flags); @@ -433,14 +440,20 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, } /** Given a list of strings in lst, set the DIGEST_LEN-byte digest at - * digest_out to the hash of the concatenation of those strings. */ + * digest_out to the hash of the concatenation of those strings. DOCDOC + * new arguments. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -655,12 +668,15 @@ networkstatus_compute_consensus(smartlist_t *votes, int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; const routerstatus_format_type_t rs_format = flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -762,8 +778,12 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_snprintf(buf, sizeof(buf), "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, tor_strdup(buf)); if (consensus_method >= 2) { tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", @@ -1285,25 +1305,36 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1316,9 +1347,15 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, DIGEST_LEN, + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1488,12 +1525,21 @@ networkstatus_add_detached_signatures(networkstatus_t *target, /** DOCDOC */ static char * -networkstatus_format_signatures(networkstatus_t *consensus) +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) { smartlist_t *elements; char buf[4096]; char *result = NULL; int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; + + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; elements = smartlist_create(); @@ -1501,14 +1547,25 @@ networkstatus_format_signatures(networkstatus_t *consensus) SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { char sk[HEX_DIGEST_LEN+1]; char id[HEX_DIGEST_LEN+1]; - if (!sig->signature || sig->bad_signature || sig->alg != DIGEST_SHA1) + if (!sig->signature || sig->bad_signature) continue; ++n_sigs; base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); + } else { + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); + } smartlist_add(elements, tor_strdup(buf)); base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); @@ -1525,17 +1582,28 @@ networkstatus_format_signatures(networkstatus_t *consensus) } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on consensus. */ + * corresponding to the signatures on consensuses, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); - tor_assert(consensus->flavor == FLAV_NS); + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1544,10 +1612,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->digests.d[DIGEST_SHA1], DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1557,12 +1626,46 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - { - char *sigs = networkstatus_format_signatures(consensus); - if (!sigs) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) + continue; + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); + smartlist_add(elements, tor_strdup(buf)); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); goto err; + } smartlist_add(elements, sigs); - } + } SMARTLIST_FOREACH_END(ns); + + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); result = smartlist_join_strings(elements, "", 0, NULL); err: @@ -1571,6 +1674,23 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) return result; } +/** DOCDOC */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in s. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) @@ -1812,15 +1932,6 @@ static smartlist_t *pending_vote_list = NULL; * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** DOCDOC*/ -struct pending_consensus_t { - /** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ - char *body; - /** The parsed in-progress consensus document. */ - networkstatus_t *consensus; -}; - static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; /** The detached signatures for the consensus that we're currently @@ -2277,9 +2388,8 @@ dirvote_compute_consensuses(void) } } - /* XXXX NMNM NM NM wrong. */ - signatures = - networkstatus_get_detached_signatures(pending[FLAV_NS].consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); @@ -2364,7 +2474,7 @@ dirvote_add_signatures_to_pending_consensus( if (r >= 1) { char *new_signatures = - networkstatus_format_signatures(pc->consensus); + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; if (!new_signatures) { @@ -2389,7 +2499,6 @@ dirvote_add_signatures_to_pending_consensus( tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); *msg_out = "Signatures added"; } else if (r == 0) { *msg_out = "Signatures ignored"; @@ -2414,6 +2523,7 @@ dirvote_add_signatures_to_all_pending_consensuses( ns_detached_signatures_t *sigs; tor_assert(detached_signatures_body); tor_assert(msg_out); + tor_assert(pending_consensus_signatures); if (!(sigs = networkstatus_parse_detached_signatures( detached_signatures_body, NULL))) { @@ -2438,10 +2548,10 @@ dirvote_add_signatures_to_all_pending_consensuses( goto err; } - /* Still not right XXXX NM NM*/ - if (pending_consensuses[FLAV_NS].consensus) { - char *new_detached = networkstatus_get_detached_signatures( - pending_consensuses[FLAV_NS].consensus); + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); if (new_detached) { tor_free(pending_consensus_signatures); pending_consensus_signatures = new_detached; diff --git a/src/or/or.h b/src/or/or.h index b8ccac9239..d9b10c663d 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3838,7 +3838,7 @@ char *networkstatus_compute_consensus(smartlist_t *votes, int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); void ns_detached_signatures_free(ns_detached_signatures_t *s); /* cert manipulation */ diff --git a/src/test/test_dir.c b/src/test/test_dir.c index b1fac683cc..13b46c63c3 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -559,6 +559,21 @@ generate_ri_from_rs(const vote_routerstatus_t *vrs) return r; } +/** Helper: get a detached signatures document for a single FLAV_NS + * consensus. */ +static char * +get_detached_sigs(networkstatus_t *ns) +{ + char *r; + smartlist_t *sl; + tor_assert(ns && ns->flavor == FLAV_NS); + sl = smartlist_create(); + smartlist_add(sl,ns); + r = networkstatus_get_detached_signatures(sl); + smartlist_free(sl); + return r; +} + /** Run unit tests for generating and parsing V3 consensus networkstatus * documents. */ static void @@ -994,7 +1009,7 @@ test_dir_v3_networkstatus(void) test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); + detached_text1 = get_detached_sigs(con3); tor_assert(detached_text1); /* Try to parse it. */ dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); @@ -1020,10 +1035,10 @@ test_dir_v3_networkstatus(void) } /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); + detached_text2 = get_detached_sigs(con2); test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); + detached_text2 = get_detached_sigs(con2); //printf("\n<%s>\n", detached_text2); dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); test_assert(dsig2); From 0bce0161dded650ac6fa665a7b861d6faac9e91c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 23 Sep 2009 11:45:54 -0400 Subject: [PATCH 32/81] Revise proposal 162: SHA256(x), not SHA256(SHA256(x)) The point of doing SHA256 twice is, generally, is to prevent message extension attacks where an attacker who knows H(A) can calculate H(A|B). But for attaching a signature to a document, the attacker already _knows_ A, so trying to keep them from calculating H(A|B) is pointless. --- doc/spec/proposals/162-consensus-flavors.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index 56a0b0e1ab..e257205bbe 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -148,11 +148,10 @@ Spec modifications: 4.1. The "sha256" signature format. The 'SHA256' signature format for directory objects is defined as - the RSA signature of the OAEP+-padded SHA256 digest of the SHA256 - digest of the item to be signed. When checking signatures, - the signature MUST be treated as valid if the signature material - begins with SHA256(SHA256(document)); this allows us to add other - data later. + the RSA signature of the OAEP+-padded SHA256 digest of the item to + be signed. When checking signatures, the signature MUST be treated + as valid if the signature material begins with SHA256(document); + this allows us to add other data later. Considerations: From a19981725d167927d32ef2a31c7a7968be3d95b6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 23 Sep 2009 15:23:04 -0400 Subject: [PATCH 33/81] Parse detached signatures and microdesc networkstatuses correctly. --- src/or/networkstatus.c | 12 +++++ src/or/or.h | 1 + src/or/routerparse.c | 115 ++++++++++++++++++++++++++++++----------- src/test/test_dir.c | 87 +++++++++++++++++++++++++------ 4 files changed, 170 insertions(+), 45 deletions(-) diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 616726525b..93a21ebb74 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1983,6 +1983,18 @@ networkstatus_get_flavor_name(consensus_flavor_t flav) } } +/** DOCDOC return -1 on unknown */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; +} + /** If question is a string beginning with "ns/" in a format the * control interface expects for a GETINFO question, set *answer to a * newly-allocated string containing networkstatus lines for the appropriate diff --git a/src/or/or.h b/src/or/or.h index d9b10c663d..a92b0b5e30 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4211,6 +4211,7 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int getinfo_helper_networkstatus(control_connection_t *conn, const char *question, char **answer); const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); void document_signature_free(document_signature_t *sig); document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 21fe5f7d23..c6278c6efe 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -321,7 +321,7 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), @@ -1900,22 +1900,29 @@ find_start_of_next_routerstatus(const char *s) * If consensus_method is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * DOCDOC flav **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1927,7 +1934,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1948,29 +1963,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -2267,7 +2287,7 @@ networkstatus_v2_parse_from_string(const char *s) while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } smartlist_sort(ns->entries, compare_routerstatus_entries); @@ -2329,6 +2349,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) @@ -2352,6 +2374,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns = tor_malloc_zero(sizeof(networkstatus_t)); memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consenus networkstatus."); + goto err; + } + if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; if (!(cert = strstr(s, "\ndir-key-certificate-version"))) @@ -2598,7 +2636,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2608,7 +2646,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2645,10 +2684,29 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2657,11 +2715,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2671,12 +2729,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } sig = tor_malloc_zero(sizeof(document_signature_t)); memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); - sig->alg = DIGEST_SHA1; - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); tor_free(sig); goto err; } @@ -2716,7 +2774,6 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } sig->signature = tor_memdup(tok->object_body, tok->object_size); sig->signature_len = (int) tok->object_size; - } smartlist_add(v->sigs, sig); @@ -2950,10 +3007,10 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) "network-status vote.", escaped(id_hexdigest)); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sk_digest, sizeof(sk_digest), sk_hexdigest, HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " "network-status vote.", escaped(sk_hexdigest)); goto err; } @@ -2974,7 +3031,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) } sig = tor_malloc_zero(sizeof(document_signature_t)); - sig->alg = DIGEST_SHA1; + sig->alg = alg; memcpy(sig->identity_digest, id_digest, DIGEST_LEN); memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); if (tok->object_size >= INT_MAX) { diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 13b46c63c3..68dbbb45a4 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -559,16 +559,18 @@ generate_ri_from_rs(const vote_routerstatus_t *vrs) return r; } -/** Helper: get a detached signatures document for a single FLAV_NS - * consensus. */ +/** Helper: get a detached signatures document for one or two + * consensuses. */ static char * -get_detached_sigs(networkstatus_t *ns) +get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2) { char *r; smartlist_t *sl; tor_assert(ns && ns->flavor == FLAV_NS); sl = smartlist_create(); smartlist_add(sl,ns); + if (ns2) + smartlist_add(sl,ns2); r = networkstatus_get_detached_signatures(sl); smartlist_free(sl); return r; @@ -587,7 +589,8 @@ test_dir_v3_networkstatus(void) time_t now = time(NULL); networkstatus_voter_info_t *voter; document_signature_t *sig; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; + networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL, + *con_md=NULL; vote_routerstatus_t *vrs; routerstatus_t *rs; char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; @@ -596,7 +599,9 @@ test_dir_v3_networkstatus(void) /* For generating the two other consensuses. */ char *detached_text1=NULL, *detached_text2=NULL; char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; + char *consensus_text_md2=NULL, *consensus_text_md3=NULL; + char *consensus_text_md=NULL; + networkstatus_t *con2=NULL, *con_md2=NULL, *con3=NULL, *con_md3=NULL; ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; /* Parse certificates and keys. */ @@ -882,6 +887,17 @@ test_dir_v3_networkstatus(void) test_assert(con); //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", // v1_text, v2_text, v3_text); + consensus_text_md = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_MICRODESC); + test_assert(consensus_text_md); + con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + NS_TYPE_CONSENSUS); + test_assert(con_md); + test_eq(con_md->flavor, FLAV_MICRODESC); /* Check consensus contents. */ test_assert(con->type == NS_TYPE_CONSENSUS); @@ -956,19 +972,11 @@ test_dir_v3_networkstatus(void) test_assert(rs->is_valid); test_assert(!rs->is_named); /* XXXX check version */ - // x231 - // x213 /* Check signatures. the first voter is a pseudo-entry with a legacy key. * The second one hasn't signed. The fourth one has signed: validate it. */ voter = smartlist_get(con->voters, 1); test_eq(smartlist_len(voter->sigs), 0); -#if 0 - sig = smartlist_get(voter->sigs, 1); - test_assert(!sig->signature); - test_assert(!sig->good_signature); - test_assert(!sig->bad_signature); -#endif voter = smartlist_get(con->voters, 3); test_eq(smartlist_len(voter->sigs), 1); @@ -990,26 +998,45 @@ test_dir_v3_networkstatus(void) cert2->identity_key, sign_skey_2, NULL,NULL, FLAV_NS); + consensus_text_md2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_MICRODESC); smartlist_shuffle(votes); consensus_text3 = networkstatus_compute_consensus(votes, 3, cert1->identity_key, sign_skey_1, NULL,NULL, FLAV_NS); + consensus_text_md3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_MICRODESC); test_assert(consensus_text2); test_assert(consensus_text3); + test_assert(consensus_text_md2); + test_assert(consensus_text_md3); con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, NS_TYPE_CONSENSUS); con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, NS_TYPE_CONSENSUS); + con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + NS_TYPE_CONSENSUS); + con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + NS_TYPE_CONSENSUS); test_assert(con2); test_assert(con3); + test_assert(con_md2); + test_assert(con_md3); /* All three should have the same digest. */ test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md2->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md3->digests, sizeof(digests_t)); + /* Extract a detached signature from con3. */ - detached_text1 = get_detached_sigs(con3); + detached_text1 = get_detached_sigs(con3, con_md3); tor_assert(detached_text1); /* Try to parse it. */ dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); @@ -1024,6 +1051,11 @@ test_dir_v3_networkstatus(void) test_assert(dsig_digests); test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], DIGEST_LEN); + dsig_digests = strmap_get(dsig1->digests, "microdesc"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA256], + con_md3->digests.d[DIGEST_SHA256], + DIGEST256_LEN); } { smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); @@ -1032,13 +1064,24 @@ test_dir_v3_networkstatus(void) sig = smartlist_get(dsig_signatures, 0); test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA1); + + dsig_signatures = strmap_get(dsig1->signatures, "microdesc"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA256); } /* Try adding it to con2. */ - detached_text2 = get_detached_sigs(con2); + detached_text2 = get_detached_sigs(con2,con_md2); test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); tor_free(detached_text2); - detached_text2 = get_detached_sigs(con2); + test_eq(1, networkstatus_add_detached_signatures(con_md2, dsig1, &msg)); + tor_free(detached_text2); + detached_text2 = get_detached_sigs(con2,con_md2); //printf("\n<%s>\n", detached_text2); dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); test_assert(dsig2); @@ -1052,6 +1095,9 @@ test_dir_v3_networkstatus(void) */ test_eq(2, smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, + "microdesc"))); /* Try adding to con2 twice; verify that nothing changes. */ test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); @@ -1075,6 +1121,7 @@ test_dir_v3_networkstatus(void) tor_free(v2_text); tor_free(v3_text); tor_free(consensus_text); + tor_free(consensus_text_md); if (vote) networkstatus_vote_free(vote); @@ -1086,6 +1133,8 @@ test_dir_v3_networkstatus(void) networkstatus_vote_free(v3); if (con) networkstatus_vote_free(con); + if (con_md) + networkstatus_vote_free(con_md); if (sign_skey_1) crypto_free_pk_env(sign_skey_1); if (sign_skey_2) @@ -1103,12 +1152,18 @@ test_dir_v3_networkstatus(void) tor_free(consensus_text2); tor_free(consensus_text3); + tor_free(consensus_text_md2); + tor_free(consensus_text_md3); tor_free(detached_text1); tor_free(detached_text2); if (con2) networkstatus_vote_free(con2); if (con3) networkstatus_vote_free(con3); + if (con_md2) + networkstatus_vote_free(con_md2); + if (con_md3) + networkstatus_vote_free(con_md3); if (dsig1) ns_detached_signatures_free(dsig1); if (dsig2) From 851a980065e6b2df8d7cb35a22d0675b8918214b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Oct 2009 17:06:01 -0400 Subject: [PATCH 34/81] Actually remember all the consensus types when we are done generating them. --- src/or/control.c | 2 +- src/or/directory.c | 2 +- src/or/dirserv.c | 48 ++++--- src/or/dirvote.c | 35 +++-- src/or/networkstatus.c | 305 +++++++++++++++++++++++++++-------------- src/or/or.h | 17 ++- 6 files changed, 269 insertions(+), 140 deletions(-) diff --git a/src/or/control.c b/src/or/control.c index 67ee37ae52..554aef6b25 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1506,7 +1506,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } diff --git a/src/or/directory.c b/src/or/directory.c index 0feddcbae4..acee78f030 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1623,7 +1623,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } log_info(LD_DIR,"Received consensus directory (size %d) from server " "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 17fff6110c..57d1a8d827 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -41,7 +41,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -1211,14 +1211,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of d with the value of * directory published on when, unless when is older than @@ -1386,17 +1386,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * networkstatus, published at published. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type flavor_name that + * we're serving with networkstatus, published at published. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1579,9 +1588,9 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -3130,8 +3139,8 @@ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) @@ -3467,6 +3476,9 @@ dirserv_free_all(void) digestmap_free(cached_v2_networkstatus, _free_cached_dir); cached_v2_networkstatus = NULL; } - cached_dir_decref(cached_v3_networkstatus); + if (cached_consensuses) { + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; + } } diff --git a/src/or/dirvote.c b/src/or/dirvote.c index d64e4d53a2..e50a875411 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -421,13 +421,13 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, microdesc_digest256_out) { smartlist_t *digests = smartlist_create(); const char *best_microdesc_digest; - SMARTLIST_FOREACH(votes, vote_routerstatus_t *, rs, { + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { char d[DIGEST256_LEN]; if (compare_vote_rs(rs, most)) continue; if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method)) smartlist_add(digests, tor_memdup(d, sizeof(d))); - }); + } SMARTLIST_FOREACH_END(rs); smartlist_sort_digests256(digests); best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); if (best_microdesc_digest) @@ -2602,20 +2602,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensuses[FLAV_NS].consensus || - networkstatus_check_consensus_signature( - pending_consensuses[FLAV_NS].consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; - /* XXXXXX NMNMNM */ - if (networkstatus_set_current_consensus( - pending_consensuses[FLAV_NS].body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } + + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 93a21ebb74..6481499717 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -35,16 +35,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +62,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +95,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +104,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +178,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +187,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ - - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); } - tor_free(s); + tor_free(s); + } + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +234,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -1123,27 +1146,32 @@ static void update_consensus_networkstatus_downloads(time_t now) { or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ if (authdir_mode_v3(options)) return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1159,7 +1187,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1265,10 +1294,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1276,7 +1309,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1445,16 +1479,29 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1464,32 +1511,70 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if (c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(get_options())) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + result = -1; + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { /* XXX022 when we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - !memcmp(&c->digests, ¤t_consensus->digests, sizeof(c->digests))) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + !memcmp(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1498,16 +1583,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus) + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1536,56 +1622,65 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (directory_caches_dir_info(get_options())) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + current_valid_after); + } - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + c = NULL; /* Prevent free. */ + + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(get_options(), now); + routerstatus_list_update_named_server_map(); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); - if (ftime_definitely_before(now, current_consensus->valid_after)) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; @@ -1616,13 +1711,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; iconsensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -2046,6 +2145,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); @@ -2060,11 +2160,14 @@ networkstatus_free_all(void) networkstatus_vote_free(current_consensus); current_consensus = NULL; } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } - tor_free(consensus_waiting_for_certs_body); if (named_server_map) { strmap_free(named_server_map, _tor_free); } diff --git a/src/or/or.h b/src/or/or.h index a92b0b5e30..b2e16a3b91 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1282,6 +1282,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of dir (not counting its NUL). */ size_t dir_z_len; /**< Length of dir_z. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1692,6 +1693,9 @@ typedef enum { FLAV_MICRODESC = 1, } consensus_flavor_t; +/** DOCDOC */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + /** DOCDOC */ #define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) @@ -3758,14 +3762,16 @@ int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); void directory_set_dirty(void); cached_dir_t *dirserv_get_directory(void); cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_directory(const char *directory, time_t when, int is_running_routers); void dirserv_set_cached_networkstatus_v2(const char *directory, const char *identity, time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); void dirserv_clear_old_v1_info(time_t now); void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); @@ -4194,7 +4200,10 @@ networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); #define NSSET_WAS_WAITING_FOR_CERTS 2 #define NSSET_DONT_DOWNLOAD_CERTS 4 #define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); void networkstatus_note_certs_arrived(void); void routers_update_all_from_networkstatus(time_t now, int dir_version); void routerstatus_list_update_from_consensus_networkstatus(time_t now); From e26a79ca8aa0df0bd03b69ef18b49fb94b22943b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Oct 2009 13:57:51 -0400 Subject: [PATCH 35/81] Make start_writing_to_stdio_file() respect O_BINARY. --- src/common/util.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/common/util.c b/src/common/util.c index 139c1aaad8..06636b8274 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1736,7 +1736,8 @@ write_str_to_file(const char *fname, const char *str, int bin) struct open_file_t { char *tempname; /**< Name of the temporary file. */ char *filename; /**< Name of the original file. */ - int rename_on_close; /**< Are we using the temporary file or not? */ + unsigned rename_on_close:1; /**< Are we using the temporary file or not? */ + unsigned binary:1; /**< Did we open in binary mode? */ int fd; /**< fd for the open file. */ FILE *stdio_file; /**< stdio wrapper for fd. */ }; @@ -1792,6 +1793,8 @@ start_writing_to_file(const char *fname, int open_flags, int mode, open_flags &= ~O_EXCL; new_file->rename_on_close = 1; } + if (open_flags & O_BINARY) + new_file->binary = 1; if ((new_file->fd = open(open_name, open_flags, mode)) < 0) { log(LOG_WARN, LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", @@ -1830,7 +1833,8 @@ fdopen_file(open_file_t *file_data) if (file_data->stdio_file) return file_data->stdio_file; tor_assert(file_data->fd >= 0); - if (!(file_data->stdio_file = fdopen(file_data->fd, "a"))) { + if (!(file_data->stdio_file = fdopen(file_data->fd, + file_data->binary?"ab":"a"))) { log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename, file_data->fd, strerror(errno)); } From d61b5df9c1bed57cb39888a1f256cf6c234c29eb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Oct 2009 16:05:08 -0400 Subject: [PATCH 36/81] Fix various bugs in microdescriptor caching. --- src/or/main.c | 1 + src/or/microdesc.c | 105 ++++++++++++++++++++++++++++++++--------- src/or/networkstatus.c | 2 +- src/or/or.h | 1 + 4 files changed, 86 insertions(+), 23 deletions(-) diff --git a/src/or/main.c b/src/or/main.c index 25182919ae..9605b2bca8 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1985,6 +1985,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0128fbbab2..2533564dd5 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -40,19 +40,28 @@ HT_GENERATE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq, 0.6, _tor_malloc, _tor_realloc, _tor_free); +/* returns n bytes written */ static int -dump_microdescriptor(FILE *f, microdesc_t *md) +dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) { + int r = 0; /* XXXX drops unkown annotations. */ if (md->last_listed) { char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; format_iso_time(buf, md->last_listed); - fprintf(f, "@last-listed %s\n", buf); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; } md->off = (off_t) ftell(f); fwrite(md->body, 1, md->bodylen, f); - return 0; + r += md->bodylen; + return r; } static microdesc_cache_t *the_microdesc_cache = NULL; @@ -72,7 +81,7 @@ get_microdesc_cache(void) } /* There are three sources of microdescriptors: - 1) Generated us while acting as a directory authority. + 1) Generated by us while acting as a directory authority. 2) Loaded from the cache on disk. 3) Downloaded. */ @@ -98,7 +107,8 @@ microdescs_add_to_cache(microdesc_cache_t *cache, return added; } -/* Returns list of added microdesc_t. Frees any not added. */ +/* Returns list of added microdesc_t. Frees any not added. Updates last_listed. + */ smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, @@ -108,13 +118,17 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, open_file_t *open_file = NULL; FILE *f = NULL; // int n_added = 0; + size_t size = 0; if (where == SAVED_NOWHERE && !no_save) { - f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND, + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, 0600, &open_file); - if (!f) - log_warn(LD_DIR, "Couldn't append to journal in %s", - cache->journal_fname); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } } added = smartlist_create(); @@ -131,8 +145,10 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, /* Okay, it's a new one. */ if (f) { - dump_microdescriptor(f, md); + int annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; } else { md->saved_location = where; } @@ -143,7 +159,18 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_add(added, md); } SMARTLIST_FOREACH_END(md); - finish_writing_to_file(open_file); /*XXX Check me.*/ + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + { + size_t old_content_len = + cache->cache_content ? cache->cache_content->size : 0; + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len * 2) { + microdesc_cache_rebuild(cache); + } + } + return added; } @@ -152,9 +179,11 @@ microdesc_cache_clear(microdesc_cache_t *cache) { microdesc_t **entry, **next; for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); - microdesc_free(*entry); + microdesc_free(md); } + HT_CLEAR(microdesc_map, &cache->map); if (cache->cache_content) { tor_munmap_file(cache->cache_content); cache->cache_content = NULL; @@ -176,8 +205,10 @@ microdesc_cache_reload(microdesc_cache_t *cache) if (mm) { added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0); - total += smartlist_len(added); - smartlist_free(added); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } } journal_content = read_file_to_str(cache->journal_fname, @@ -186,8 +217,10 @@ microdesc_cache_reload(microdesc_cache_t *cache) added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0); - total += smartlist_len(added); - smartlist_free(added); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } tor_free(journal_content); } log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", @@ -202,8 +235,16 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) FILE *f; microdesc_t **mdp; smartlist_t *wrote; + int size; + off_t off = 0; + int orig_size, new_size; - f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE, + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, 0600, &open_file); if (!f) return -1; @@ -212,15 +253,17 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) HT_FOREACH(mdp, microdesc_map, &cache->map) { microdesc_t *md = *mdp; + int annotation_len; if (md->no_save) continue; - dump_microdescriptor(f, md); + size = dump_microdescriptor(f, md, &annotation_len); + md->off = off + annotation_len; + off += size; if (md->saved_location != SAVED_IN_CACHE) { tor_free(md->body); md->saved_location = SAVED_IN_CACHE; } - smartlist_add(wrote, md); } @@ -229,21 +272,29 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) if (cache->cache_content) tor_munmap_file(cache->cache_content); cache->cache_content = tor_mmap_file(cache->cache_fname); + if (!cache->cache_content && smartlist_len(wrote)) { log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", cache->cache_fname); + smartlist_free(wrote); return -1; } SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { - if (md->no_save) - continue; tor_assert(md->saved_location == SAVED_IN_CACHE); md->body = (char*)cache->cache_content->data + md->off; tor_assert(!memcmp(md->body, "onion-key", 9)); - } SMARTLIST_FOREACH_END(wrote); + } SMARTLIST_FOREACH_END(md); smartlist_free(wrote); + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + + new_size = (int)cache->cache_content->size; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + return 0; } @@ -265,3 +316,13 @@ microdesc_free(microdesc_t *md) tor_free(md); } +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 6481499717..37838f4b08 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1664,7 +1664,7 @@ networkstatus_set_current_consensus(const char *consensus, dirserv_set_cached_consensus_networkstatus(consensus, flavor, &c->digests, - current_valid_after); + c->valid_after); } if (flav == USABLE_CONSENSUS_FLAVOR) { diff --git a/src/or/or.h b/src/or/or.h index b2e16a3b91..63d2eee9da 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4123,6 +4123,7 @@ int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache); void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); /********************************* networkstatus.c *********************/ From 200c39b66cc6e157e78a67481b85e237ccffbef3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 15 Oct 2009 16:06:00 -0400 Subject: [PATCH 37/81] Document the microdescriptor code better. --- src/common/crypto.c | 3 ++- src/or/dirserv.c | 3 ++- src/or/dirvote.c | 48 ++++++++++++++++++++++++++++++------------ src/or/microdesc.c | 46 ++++++++++++++++++++++++++++++++-------- src/or/networkstatus.c | 3 ++- src/or/or.h | 34 +++++++++++++++++++++++++----- src/or/routerparse.c | 25 ++++++++++++++-------- 7 files changed, 122 insertions(+), 40 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index f335f99aba..4ea4492898 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1481,7 +1481,8 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg) } } -/** DOCDOC */ +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ int crypto_digest_algorithm_parse_name(const char *name) { diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 57d1a8d827..251c35dd38 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1885,7 +1885,8 @@ version_from_platform(const char *platform) * The format argument has three possible values: * NS_V2 - Output an entry suitable for a V2 NS opinion document * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry - * NS_V3_CONSENSUS_MICRODESC - DOCDOC + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. * NS_V3_VOTE - Output a complete V3 NS vote * NS_CONTROL_PORT - Output a NS document for the control port */ diff --git a/src/or/dirvote.c b/src/or/dirvote.c index e50a875411..eb3c7d2cfa 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,7 +11,9 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -/** DOCDOC*/ +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ typedef struct pending_consensus_t { /** The body of the consensus that we're currently building. Once we * have it built, it goes into dirserv.c */ @@ -264,7 +266,8 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } -/** DOCDOC */ +/** Return the signature made by voter using the algorithm + * alg, or NULL if none is found. */ document_signature_t * voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, digest_algorithm_t alg) @@ -425,7 +428,8 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, char d[DIGEST256_LEN]; if (compare_vote_rs(rs, most)) continue; - if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method)) + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) smartlist_add(digests, tor_memdup(d, sizeof(d))); } SMARTLIST_FOREACH_END(rs); smartlist_sort_digests256(digests); @@ -439,9 +443,9 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, return most; } -/** Given a list of strings in lst, set the DIGEST_LEN-byte digest at - * digest_out to the hash of the concatenation of those strings. DOCDOC - * new arguments. */ +/** Given a list of strings in lst, set the len_out-byte digest + * at digest_out to the hash of the concatenation of those strings, + * computed with the algorithm alg. */ static void hash_list_members(char *digest_out, size_t len_out, smartlist_t *lst, digest_algorithm_t alg) @@ -1523,7 +1527,11 @@ networkstatus_add_detached_signatures(networkstatus_t *target, return r; } -/** DOCDOC */ +/** Return a newly allocated string containing all the signatures on + * consensus by all voters. If for_detached_signatures is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ static char * networkstatus_format_signatures(networkstatus_t *consensus, int for_detached_signatures) @@ -1674,7 +1682,9 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses) return result; } -/** DOCDOC */ +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the n_flavors-element array at + * pending. */ static char * get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, int n_flavors) @@ -2049,7 +2059,8 @@ dirvote_fetch_missing_signatures(void) 0, NULL); } -/** DOCDOC */ +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ static void dirvote_clear_pending_consensuses(void) { @@ -2775,7 +2786,10 @@ dirvote_create_microdescriptor(const routerinfo_t *ri) /** Cached space-separated string to hold */ static char *microdesc_consensus_methods = NULL; -/** DOCDOC */ +/** Format the appropriate vote line to describe the microdescriptor md + * in a consensus vote document. Write it into the out_len-byte buffer + * in out. Return -1 on failure and the number of characters written + * on success. */ int dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md) @@ -2798,19 +2812,25 @@ dirvote_format_microdesc_vote_line(char *out, size_t out_len, return strlen(out); } -/** DOCDOC */ +/** If vrs has a hash made for the consensus method method with + * the digest algorithm alg, decode it and copy it into + * digest256_out and return 0. Otherwise return -1. */ int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, - int method) + int method, + digest_algorithm_t alg) { /* XXXX only returns the sha256 method. */ const vote_microdesc_hash_t *h; char mstr[64]; size_t mlen; + char dstr[64]; tor_snprintf(mstr, sizeof(mstr), "%d", method); mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); for (h = vrs->microdesc; h; h = h->next) { const char *cp = h->microdesc_hash_line; @@ -2824,10 +2844,10 @@ vote_routerstatus_find_microdesc_hash(char *digest256_out, char buf[BASE64_DIGEST256_LEN+1]; /* XXXX ignores extraneous stuff if the digest is too long. This * seems harmless enough, right? */ - cp = strstr(cp, " sha256="); + cp = strstr(cp, dstr); if (!cp) return -1; - cp += strlen(" sha256="); + cp += strlen(dstr); strlcpy(buf, cp, sizeof(buf)); return digest256_from_base64(digest256_out, buf); } diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 2533564dd5..85070650e8 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -3,20 +3,26 @@ #include "or.h" -/** DOCDOC everything here. */ - -#define MICRODESC_IN_CONSENSUS 1 -#define MICRODESC_IN_VOTE 2 - +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ HT_HEAD(microdesc_map, microdesc_t) map; + /** Name of the cache file. */ char *cache_fname; + /** Name of the journal file. */ char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ size_t journal_len; }; +/** Helper: computes a hash of md to place it in a hash table. */ static INLINE unsigned int _microdesc_hash(microdesc_t *md) { @@ -28,6 +34,7 @@ _microdesc_hash(microdesc_t *md) #endif } +/** Helper: compares a and for equality for hash-table purposes. */ static INLINE int _microdesc_eq(microdesc_t *a, microdesc_t *b) { @@ -40,7 +47,10 @@ HT_GENERATE(microdesc_map, microdesc_t, node, _microdesc_hash, _microdesc_eq, 0.6, _tor_malloc, _tor_realloc, _tor_free); -/* returns n bytes written */ +/** Write the body of md into f, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *annotation_len_out to the number of bytes written as + * annotations. */ static int dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) { @@ -64,8 +74,11 @@ dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) return r; } +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ static microdesc_cache_t *the_microdesc_cache = NULL; +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ microdesc_cache_t * get_microdesc_cache(void) { @@ -86,7 +99,12 @@ get_microdesc_cache(void) 3) Downloaded. */ -/* Returns list of added microdesc_t. */ +/** Decode the microdescriptors from the string starting at s and + * ending at eos, and store them in cache. If no-save, + * mark them as non-writable to disk. If where is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * SAVED_NOWHERE, do not allow annotations. Return a list of the added + * microdescriptors. */ smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, @@ -107,8 +125,9 @@ microdescs_add_to_cache(microdesc_cache_t *cache, return added; } -/* Returns list of added microdesc_t. Frees any not added. Updates last_listed. - */ +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of descriptors that it does + * not add. */ smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, @@ -174,6 +193,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, return added; } +/** Remove every microdescriptor in cache. */ void microdesc_cache_clear(microdesc_cache_t *cache) { @@ -190,6 +210,8 @@ microdesc_cache_clear(microdesc_cache_t *cache) } } +/** Reload the contents of cache from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ int microdesc_cache_reload(microdesc_cache_t *cache) { @@ -228,6 +250,9 @@ microdesc_cache_reload(microdesc_cache_t *cache) return 0; } +/** Regenerate the main cache file for cache, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. */ int microdesc_cache_rebuild(microdesc_cache_t *cache) { @@ -298,6 +323,8 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) return 0; } +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ void microdesc_free(microdesc_t *md) { @@ -316,6 +343,7 @@ microdesc_free(microdesc_t *md) tor_free(md); } +/** Free all storage held in the microdesc.c module. */ void microdesc_free_all(void) { diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 37838f4b08..e9e8663062 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -2082,7 +2082,8 @@ networkstatus_get_flavor_name(consensus_flavor_t flav) } } -/** DOCDOC return -1 on unknown */ +/** Return the consensus_flavor_t value for the flavor called flavname, + * or -1 if the flavor is not recongized. */ int networkstatus_parse_flavor_name(const char *flavname) { diff --git a/src/or/or.h b/src/or/or.h index 63d2eee9da..21e90a4cdf 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1559,27 +1559,48 @@ typedef struct routerstatus_t { } routerstatus_t; -/**DOCDOC*/ +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ HT_ENTRY(microdesc_t) node; /* Cache information */ + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ time_t last_listed; + /** Where is this microdescriptor currently stored? */ saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ off_t off; /* The string containing the microdesc. */ + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from bodylen. */ char *body; + /** The length of the microdescriptor in body. */ size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ char digest[DIGEST256_LEN]; /* Fields in the microdescriptor. */ + /** As routerinfo_t.onion_pkey */ crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ smartlist_t *family; + /** Encoded exit policy summary */ char *exitsummary; /**< exit policy summary - * XXX this probably should not stay a string. */ } microdesc_t; @@ -1687,16 +1708,18 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; -/** DOCDOC */ +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ typedef enum { FLAV_NS = 0, FLAV_MICRODESC = 1, } consensus_flavor_t; -/** DOCDOC */ +/** Which consensus flavor do we actually want to use to build circuits? */ #define USABLE_CONSENSUS_FLAVOR FLAV_NS -/** DOCDOC */ +/** How many different consensus flavors are there? */ #define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) /** A common structure to hold a v3 network status vote, or a v3 network @@ -3892,7 +3915,8 @@ int dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md); int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, - int method); + int method, + digest_algorithm_t alg); document_signature_t *voter_get_sig_by_algorithm( const networkstatus_voter_info_t *voter, digest_algorithm_t alg); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index c6278c6efe..1f89cffa01 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -640,7 +640,8 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) DIGEST_SHA1); } -/** DOCDOC */ +/** Set digests to all the digests of the consensus document in + * s */ int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) { @@ -677,8 +678,6 @@ router_get_extrainfo_hash(const char *s, char *digest) * surround it with -----BEGIN/END----- pairs, and write it to the * buf_len-byte buffer at buf. Return 0 on success, -1 on * failure. - * - * DOCDOC alg */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, @@ -1901,7 +1900,7 @@ find_start_of_next_routerstatus(const char *s) * consensus, and we should parse it according to the method used to * make that consensus. * - * DOCDOC flav + * Parse according to the syntax used by the consensus flavor flav. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, @@ -2832,7 +2831,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } -/** DOCDOC */ +/** Return the digests_t that holds the digests of the + * flavor_name-flavored networkstatus according to the detached + * signatures document sigs, allocating a new digests_t as neeeded. */ static digests_t * detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) { @@ -2844,7 +2845,9 @@ detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) return d; } -/** DOCDOC */ +/** Return the list of signatures of the flavor_name-flavored + * networkstatus according to the detached signatures document sigs, + * allocating a new digests_t as neeeded. */ static smartlist_t * detached_get_signatures(ns_detached_signatures_t *sigs, const char *flavor_name) @@ -3719,8 +3722,8 @@ router_get_hashes_impl(const char *s, digests_t *digests, return 0; } -/** DOCDOC Assuming that s starts with a microdesc, return the start of the - * *NEXT* one. */ +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ static const char * find_start_of_next_microdesc(const char *s, const char *eos) { @@ -3771,7 +3774,11 @@ find_start_of_next_microdesc(const char *s, const char *eos) #undef NEXT_LINE } -/**DOCDOC*/ +/** Parse as many microdescriptors as are found from the string starting at + * s and ending at eos. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If copy_body is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, int copy_body) From a007a7c6ba39c77b36faa1e1c4864252ec1af8b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 18 Oct 2009 15:09:56 -0400 Subject: [PATCH 38/81] Revise consensus-flavors proposal to better URLs. The old flavored consensus URL format made it harder to decode URLs based on their prefixes, and didn't take into account our "only give it to me if it's signed by enough authorities" stuff. --- doc/spec/proposals/162-consensus-flavors.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt index e257205bbe..e3b697afee 100644 --- a/doc/spec/proposals/162-consensus-flavors.txt +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -72,9 +72,11 @@ Spec modifications: design. In addition to the consensus currently served at - /tor/status-vote/(current|next)/consensus.z , authorities serve - another consensus of each flavor "F" from the location - /tor/status-vote/(current|next)/F/consensus.z. + /tor/status-vote/(current|next)/consensus.z and + /tor/status-vote/(current|next)/consensus/+++....z , + authorities serve another consensus of each flavor "F" from the + locations /tor/status-vote/(current|next)/consensus-F.z. and + /tor/status-vote/(current|next)/consensus-F/+....z. When caches serve these documents, they do so from the same locations. From bb22d8fc45f143666f0ee676e0aeae200104a421 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 18 Oct 2009 15:45:57 -0400 Subject: [PATCH 39/81] Add functions to serve microdescs and flavored consensuses. --- src/or/directory.c | 166 ++++++++++++++++++++++++++++++++------------ src/or/dirserv.c | 93 ++++++++++++++++++++++--- src/or/microdesc.c | 35 ++++++++++ src/or/or.h | 19 ++++- src/or/routerlist.c | 2 +- src/test/test_dir.c | 131 +++++++++++++++++++++++++++++++++- 6 files changed, 385 insertions(+), 61 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index acee78f030..01647ceaa2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -92,6 +92,7 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ @@ -610,7 +611,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -647,7 +648,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -1564,7 +1565,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1717,7 +1718,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -2328,7 +2329,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; @@ -2504,6 +2505,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2518,19 +2520,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2644,7 +2671,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2697,6 +2725,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2778,7 +2841,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2788,7 +2852,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -3523,19 +3588,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory resource request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into fp_out. If compressed_out is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3547,22 +3630,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3572,26 +3658,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 251c35dd38..6ce6a461e1 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -2870,7 +2870,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2935,10 +2936,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -3003,7 +3006,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -3016,7 +3020,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -3132,17 +3137,20 @@ dirserv_test_reachability(time_t now, int try_all) ctr = (ctr + 1) % 128; } -/** Given a fingerprint fp which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint fp which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; if (tor_digest_is_zero(fp) && cached_consensuses) d = strmap_get(cached_consensuses, "ns"); - else if (router_digest_is_me(fp) && the_v2_networkstatus) + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX022 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3228,6 +3236,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in fps is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3259,6 +3279,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3322,6 +3353,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3345,6 +3378,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * conn-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3452,6 +3523,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 85070650e8..e10589fec9 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -20,6 +20,11 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; }; /** Helper: computes a hash of md to place it in a hash table. */ @@ -176,6 +181,8 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, HT_INSERT(microdesc_map, &cache->map, md); smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; } SMARTLIST_FOREACH_END(md); if (f) @@ -208,6 +215,8 @@ microdesc_cache_clear(microdesc_cache_t *cache) tor_munmap_file(cache->cache_content); cache->cache_content = NULL; } + cache->total_len_seen = 0; + cache->n_seen = 0; } /** Reload the contents of cache from disk. If it is empty, load it @@ -354,3 +363,29 @@ microdesc_free_all(void) tor_free(the_microdesc_cache); } } + +/** If there is a microdescriptor in cache whose sha256 digest is + * d, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to cache since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/or.h b/src/or/or.h index 21e90a4cdf..0ec8029f46 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1171,7 +1171,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -3678,9 +3679,13 @@ void directory_initiate_command(const char *address, const tor_addr_t *addr, const char *payload, size_t payload_len, time_t if_modified_since); +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); + smartlist_t *fp_out, int *compressed_out, + int flags); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; @@ -3816,8 +3821,11 @@ int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, int dirserv_would_reject_router(routerstatus_t *rs); int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + typedef enum { NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT, NS_V3_CONSENSUS_MICRODESC @@ -4146,6 +4154,11 @@ int microdesc_cache_rebuild(microdesc_cache_t *cache); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache); +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + void microdesc_free(microdesc_t *md); void microdesc_free_all(void); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 5ae40ddd67..d5e3b924e4 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -3832,7 +3832,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 68dbbb45a4..ca55c115ac 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -361,9 +361,9 @@ test_dir_versions(void) ; } -/** Run unit tests for misc directory functions. */ +/** Run unit tests for directory fp_pair functions. */ static void -test_dir_util(void) +test_dir_fp_pairs(void) { smartlist_t *sl = smartlist_create(); fp_pair_t *pair; @@ -390,6 +390,127 @@ test_dir_util(void) smartlist_free(sl); } +static void +test_dir_split_fps(void *testdata) +{ + smartlist_t *sl = smartlist_create(); + char *mem_op_hex_tmp = NULL; + (void)testdata; + + /* Some example hex fingerprints and their base64 equivalents */ +#define HEX1 "Fe0daff89127389bc67558691231234551193EEE" +#define HEX2 "Deadbeef99999991111119999911111111f00ba4" +#define HEX3 "b33ff00db33ff00db33ff00db33ff00db33ff00d" +#define HEX256_1 \ + "f3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf" +#define HEX256_2 \ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCCc" +#define HEX256_3 \ + "0123456789ABCdef0123456789ABCdef0123456789ABCdef0123456789ABCdef" +#define B64_1 "/g2v+JEnOJvGdVhpEjEjRVEZPu4" +#define B64_2 "3q2+75mZmZERERmZmRERERHwC6Q" +#define B64_3 "sz/wDbM/8A2zP/ANsz/wDbM/8A0" +#define B64_256_1 "8/Pz8/u7vz8/Pz+7vz8/Pz+7u/Pz8/P7u/Pz8/P7u78" +#define B64_256_2 "zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw" +#define B64_256_3 "ASNFZ4mrze8BI0VniavN7wEjRWeJq83vASNFZ4mrze8" + + /* no flags set */ + dir_split_resource_into_fingerprints("A+C+B", sl, NULL, 0); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "C"); + tt_str_op(smartlist_get(sl, 2), ==, "B"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* uniq strings. */ + dir_split_resource_into_fingerprints("A+C+B+A+B+B", sl, NULL, DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "B"); + tt_str_op(smartlist_get(sl, 2), ==, "C"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2, sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* decode hex and drop weirdness. */ + dir_split_resource_into_fingerprints(HEX1"+bogus+"HEX2"+"HEX256_1, + sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX2"+"HEX256_3, + sl, NULL, DSR_HEX|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_3); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex and sort. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2"+"HEX3"+"HEX2, + sl, NULL, DSR_HEX|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex and sort */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX256_3 + "+"HEX256_1, + sl, NULL, + DSR_HEX|DSR_DIGEST256|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode base64 */ + dir_split_resource_into_fingerprints(B64_1"-"B64_2, sl, NULL, DSR_BASE64); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long base64 */ + dir_split_resource_into_fingerprints(B64_256_1"-"B64_256_2, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + dir_split_resource_into_fingerprints(B64_256_1, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 1); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(mem_op_hex_tmp); +} + static void test_dir_measured_bw(void) { @@ -1173,11 +1294,15 @@ test_dir_v3_networkstatus(void) #define DIR_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_dir_ ## name } +#define DIR(name) \ + { #name, test_dir_##name, 0, NULL, NULL } + struct testcase_t dir_tests[] = { DIR_LEGACY(nicknames), DIR_LEGACY(formats), DIR_LEGACY(versions), - DIR_LEGACY(util), + DIR_LEGACY(fp_pairs), + DIR(split_fps), DIR_LEGACY(measured_bw), DIR_LEGACY(param_voting), DIR_LEGACY(v3_networkstatus), From 740806c453c46e9c985029a644af3a6f899be205 Mon Sep 17 00:00:00 2001 From: Sebastian Hahn Date: Mon, 19 Oct 2009 01:30:46 +0200 Subject: [PATCH 40/81] Fix compile with warnings problems on Snow Leopard --- src/or/dirvote.c | 2 +- src/or/microdesc.c | 12 ++++++------ src/or/or.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index eb3c7d2cfa..ca81b2be9d 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -2790,7 +2790,7 @@ static char *microdesc_consensus_methods = NULL; * in a consensus vote document. Write it into the out_len-byte buffer * in out. Return -1 on failure and the number of characters written * on success. */ -int +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md) { diff --git a/src/or/microdesc.c b/src/or/microdesc.c index e10589fec9..7a65705088 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -56,10 +56,10 @@ HT_GENERATE(microdesc_map, microdesc_t, node, * On success, return the total number of bytes written, and set * *annotation_len_out to the number of bytes written as * annotations. */ -static int -dump_microdescriptor(FILE *f, microdesc_t *md, int *annotation_len_out) +static size_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) { - int r = 0; + size_t r = 0; /* XXXX drops unkown annotations. */ if (md->last_listed) { char buf[ISO_TIME_LEN+1]; @@ -169,7 +169,7 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, /* Okay, it's a new one. */ if (f) { - int annotation_len; + size_t annotation_len; size = dump_microdescriptor(f, md, &annotation_len); md->saved_location = SAVED_IN_JOURNAL; cache->journal_len += size; @@ -269,7 +269,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) FILE *f; microdesc_t **mdp; smartlist_t *wrote; - int size; + size_t size; off_t off = 0; int orig_size, new_size; @@ -287,7 +287,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) HT_FOREACH(mdp, microdesc_map, &cache->map) { microdesc_t *md = *mdp; - int annotation_len; + size_t annotation_len; if (md->no_save) continue; diff --git a/src/or/or.h b/src/or/or.h index 0ec8029f46..2fa4a797ad 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3919,7 +3919,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); -int dirvote_format_microdesc_vote_line(char *out, size_t out_len, +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md); int vote_routerstatus_find_microdesc_hash(char *digest256_out, const vote_routerstatus_t *vrs, From 465d4e1cd10a995cff571b42b4f008811590c314 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Oct 2009 00:30:52 -0400 Subject: [PATCH 41/81] Document some formerly undocumented functions. --- src/common/util.c | 19 +++++++++++++++---- src/or/routerlist.c | 4 +++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/common/util.c b/src/common/util.c index d05c308fe8..8f10d2175c 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -2307,7 +2307,8 @@ expand_filename(const char *filename) #define MAX_SCANF_WIDTH 9999 -/** DOCDOC */ +/** Helper: given an ASCII-encoded decimal digit, return its numeric value. + * NOTE: requires that its input be in-bounds. */ static int digit_to_num(char d) { @@ -2316,7 +2317,10 @@ digit_to_num(char d) return num; } -/** DOCDOC */ +/** Helper: Read an unsigned int from *bufp of up to width + * characters. (Handle arbitrary width if width is less than 0.) On + * success, store the result in out, advance bufp to the next + * character, and return 0. On failure, return -1. */ static int scan_unsigned(const char **bufp, unsigned *out, int width) { @@ -2343,7 +2347,9 @@ scan_unsigned(const char **bufp, unsigned *out, int width) return 0; } -/** DOCDOC */ +/** Helper: copy up to width non-space characters from bufp to + * out. Make sure out is nul-terminated. Advance bufp + * to the next non-space character or the EOS. */ static int scan_string(const char **bufp, char *out, int width) { @@ -2432,7 +2438,12 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) * and store the results in the corresponding argument fields. Differs from * sscanf in that it: Only handles %u and %Ns. Does not handle arbitrarily * long widths. %u does not consume any space. Is locale-independent. - * Returns -1 on malformed patterns. */ + * Returns -1 on malformed patterns. + * + * (As with other local-independent functions, we need this to parse data that + * is in ASCII without worrying that the C library's locale-handling will make + * miscellaneous characters look like numbers, spaces, and so on.) + */ int tor_sscanf(const char *buf, const char *pattern, ...) { diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 0a32f78a69..77b8f1d9bf 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -5034,7 +5034,9 @@ routerset_parse(routerset_t *target, const char *s, const char *description) return r; } -/** DOCDOC */ +/** Called when we change a node set, or when we reload the geoip list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ void refresh_all_country_info(void) { From d40a814f4f6cceeb4af33f92372e6f5f00e7203d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Oct 2009 00:39:20 -0400 Subject: [PATCH 42/81] Changelog for microdescriptor branch. --- ChangeLog | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ChangeLog b/ChangeLog index befb4368e4..b424837906 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,25 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? + o Major features: + - Directory authorities can now create, vote, and serve on multiple + parallel formats of directory data as part of their voting process. + This is a partial implementation of Proposal 162: "Publish the + consensus in multiple flavors." + - Directory authorities can now agree on and publish small summaries of + router information that clients can use in place of regular server + descriptors. This will eventually allow clients to use far less + bandwidth for downloading information about the network. This begins + the implementation of of Proposal 158: "Clients download a consensus + + Microdescriptors". + - The directory voting system is now extensible to use multiple hash + algorithms for signatures and resource selection. Newer formats are + signed with SHA256, with a possibility for moving to a better hash + algorithm in the future. + o Code simplifications and refactorings: - Numerous changes, bugfixes, and workarounds from Nathan Freitas to help Tor build correctly for Android phones. + - Begun converting Tor's signature and message digest logic to handle + multiple hash algorithms. o Minor bugfixes: - Fix a crash bug when trying to initialize the evdns module in From afc76a4e714a192e76281793f39c412c87964e46 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Oct 2009 23:19:42 -0400 Subject: [PATCH 43/81] Fix two bugs found by Coverity scan. One was a simple buffer overrun; the other was a high-speed pointer collision. Both were introduced by my microdescs branch. --- src/common/container.c | 10 +++++----- src/or/dirvote.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/container.c b/src/common/container.c index 4fb94d3f7c..f3540f74d8 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -470,26 +470,26 @@ smartlist_get_most_frequent(const smartlist_t *sl, const void *most_frequent = NULL; int most_frequent_count = 0; - const void **cur = NULL; + const void *cur = NULL; int i, count=0; if (!sl->num_used) return NULL; for (i = 0; i < sl->num_used; ++i) { const void *item = sl->list[i]; - if (cur && 0 == compare(cur, &item)) { + if (cur && 0 == compare(&cur, &item)) { ++count; } else { if (cur && count >= most_frequent_count) { - most_frequent = *cur; + most_frequent = cur; most_frequent_count = count; } - cur = &item; + cur = item; count = 1; } } if (cur && count >= most_frequent_count) { - most_frequent = *cur; + most_frequent = cur; most_frequent_count = count; } return (void*)most_frequent; diff --git a/src/or/dirvote.c b/src/or/dirvote.c index ca81b2be9d..5ce3fd2ca7 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -2794,7 +2794,7 @@ ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, const microdesc_t *md) { - char d64[BASE64_DIGEST256_LEN]; + char d64[BASE64_DIGEST256_LEN+1]; if (!microdesc_consensus_methods) { microdesc_consensus_methods = make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, From fa23430496f2675c2dca5dbc0c6455c404c184fe Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 26 Oct 2009 01:32:27 -0400 Subject: [PATCH 44/81] clean up the XXX comments around bug 1038 --- src/or/command.c | 6 ++++-- src/or/relay.c | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/or/command.c b/src/or/command.c index 67e463723f..8ed5a96019 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -395,8 +395,10 @@ command_process_relay_cell(cell_t *cell, or_connection_t *conn) * gotten no more than MAX_RELAY_EARLY_CELLS_PER_CIRCUIT of them. */ if (cell->command == CELL_RELAY_EARLY) { if (direction == CELL_DIRECTION_IN) { - /* XXX Allow an unlimited number of inbound relay_early cells for - * now, for hidden service compatibility. See bug 1038. -RD */ + /* Allow an unlimited number of inbound relay_early cells, + * for hidden service compatibility. There isn't any way to make + * a long circuit through inbound relay_early cells anyway. See + * bug 1038. -RD */ } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->remaining_relay_early_cells == 0) { diff --git a/src/or/relay.c b/src/or/relay.c index 653aa594cc..2151ddeb9b 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -556,9 +556,9 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, * Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ /* XXX For now, clients don't use RELAY_EARLY cells when sending - * relay cells on rendezvous circuits. See bug 1038. Eventually, - * we can take this behavior away in favor of having clients avoid - * rendezvous points running 0.2.1.3-alpha through 0.2.1.18. -RD */ + * relay cells on rendezvous circuits. See bug 1038. Once no relays + * (and thus no rendezvous points) are running 0.2.1.3-alpha through + * 0.2.1.18, we can take out that exception. -RD */ cell.command = CELL_RELAY_EARLY; --origin_circ->remaining_relay_early_cells; log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", From d2b4b49ff043df43c048b6f1f52a34dd5c4c0108 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Sun, 25 Oct 2009 23:47:05 -0700 Subject: [PATCH 45/81] Reduce log level for someone else sending us weak DH keys. See task 1114. The most plausible explanation for someone sending us weak DH keys is that they experiment with their Tor code or implement a new Tor client. Usually, we don't care about such events, especially not on warn level. If we really care about someone not following the Tor protocol, we can set ProtocolWarnings to 1. --- src/common/crypto.c | 18 +++++++++--------- src/common/crypto.h | 2 +- src/or/onion.c | 10 ++++++---- src/or/rendclient.c | 5 +++-- src/or/rendservice.c | 3 ++- src/test/test_crypto.c | 4 ++-- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index 4ea4492898..5a39a1ab0c 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -122,7 +122,7 @@ struct crypto_dh_env_t { }; static int setup_openssl_threading(void); -static int tor_check_dh_key(BIGNUM *bn); +static int tor_check_dh_key(int severity, BIGNUM *bn); /** Return the number of bytes added by padding method padding. */ @@ -1723,7 +1723,7 @@ crypto_dh_generate_public(crypto_dh_env_t *dh) crypto_log_errors(LOG_WARN, "generating DH key"); return -1; } - if (tor_check_dh_key(dh->dh->pub_key)<0) { + if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid. I guess once-in-" "the-universe chances really do happen. Trying again."); /* Free and clear the keys, so OpenSSL will actually try again. */ @@ -1770,7 +1770,7 @@ crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey, size_t pubkey_len) * See http://www.cl.cam.ac.uk/ftp/users/rja14/psandqs.ps.gz for some tips. */ static int -tor_check_dh_key(BIGNUM *bn) +tor_check_dh_key(int severity, BIGNUM *bn) { BIGNUM *x; char *s; @@ -1781,13 +1781,13 @@ tor_check_dh_key(BIGNUM *bn) init_dh_param(); BN_set_word(x, 1); if (BN_cmp(bn,x)<=0) { - log_warn(LD_CRYPTO, "DH key must be at least 2."); + log_fn(severity, LD_CRYPTO, "DH key must be at least 2."); goto err; } BN_copy(x,dh_param_p); BN_sub_word(x, 1); if (BN_cmp(bn,x)>=0) { - log_warn(LD_CRYPTO, "DH key must be at most p-2."); + log_fn(severity, LD_CRYPTO, "DH key must be at most p-2."); goto err; } BN_free(x); @@ -1795,7 +1795,7 @@ tor_check_dh_key(BIGNUM *bn) err: BN_free(x); s = BN_bn2hex(bn); - log_warn(LD_CRYPTO, "Rejecting insecure DH key [%s]", s); + log_fn(severity, LD_CRYPTO, "Rejecting insecure DH key [%s]", s); OPENSSL_free(s); return -1; } @@ -1813,7 +1813,7 @@ tor_check_dh_key(BIGNUM *bn) * where || is concatenation.) */ ssize_t -crypto_dh_compute_secret(crypto_dh_env_t *dh, +crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_bytes_out) { @@ -1828,9 +1828,9 @@ crypto_dh_compute_secret(crypto_dh_env_t *dh, if (!(pubkey_bn = BN_bin2bn((const unsigned char*)pubkey, (int)pubkey_len, NULL))) goto error; - if (tor_check_dh_key(pubkey_bn)<0) { + if (tor_check_dh_key(severity, pubkey_bn)<0) { /* Check for invalid public keys. */ - log_warn(LD_CRYPTO,"Rejected invalid g^x"); + log_fn(severity, LD_CRYPTO,"Rejected invalid g^x"); goto error; } secret_tmp = tor_malloc(crypto_dh_get_bytes(dh)); diff --git a/src/common/crypto.h b/src/common/crypto.h index c0a4526255..d9adb16f80 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -198,7 +198,7 @@ int crypto_dh_get_bytes(crypto_dh_env_t *dh); int crypto_dh_generate_public(crypto_dh_env_t *dh); int crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey_out, size_t pubkey_out_len); -ssize_t crypto_dh_compute_secret(crypto_dh_env_t *dh, +ssize_t crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_out_len); void crypto_dh_free(crypto_dh_env_t *dh); diff --git a/src/or/onion.c b/src/or/onion.c index b49a86aba3..58a51aedfe 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -253,8 +253,9 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ key_material_len = DIGEST_LEN+key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(dh, challenge, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge, + DH_KEY_LEN, key_material, + key_material_len); if (len < 0) { log_info(LD_GENERAL, "crypto_dh_compute_secret failed."); goto err; @@ -304,8 +305,9 @@ onion_skin_client_handshake(crypto_dh_env_t *handshake_state, key_material_len = DIGEST_LEN + key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(handshake_state, handshake_reply, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, + handshake_reply, DH_KEY_LEN, key_material, + key_material_len); if (len < 0) goto err; diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 0ade46807c..b7ea40eed7 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -621,8 +621,9 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; tor_assert(hop->dh_handshake_state); - if (crypto_dh_compute_secret(hop->dh_handshake_state, request, DH_KEY_LEN, - keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, hop->dh_handshake_state, + request, DH_KEY_LEN, keys, + DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_GENERAL, "Couldn't complete DH handshake."); goto err; } diff --git a/src/or/rendservice.c b/src/or/rendservice.c index daf8247e55..f00cfd44d8 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1090,7 +1090,8 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, reason = END_CIRC_REASON_INTERNAL; goto err; } - if (crypto_dh_compute_secret(dh, ptr+REND_COOKIE_LEN, DH_KEY_LEN, keys, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, ptr+REND_COOKIE_LEN, + DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't complete DH handshake"); reason = END_CIRC_REASON_INTERNAL; diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 2edb8035bb..670fd0048f 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -33,8 +33,8 @@ test_crypto_dh(void) memset(s1, 0, DH_BYTES); memset(s2, 0xFF, DH_BYTES); - s1len = crypto_dh_compute_secret(dh1, p2, DH_BYTES, s1, 50); - s2len = crypto_dh_compute_secret(dh2, p1, DH_BYTES, s2, 50); + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p2, DH_BYTES, s1, 50); + s2len = crypto_dh_compute_secret(LOG_WARN, dh2, p1, DH_BYTES, s2, 50); test_assert(s1len > 0); test_eq(s1len, s2len); test_memeq(s1, s2, s1len); From e0b2c84cde102b8202d47dccbde54245af03cb43 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 26 Oct 2009 03:16:47 -0400 Subject: [PATCH 46/81] add a changelog entry to karsten's fix for bug 1114 --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index b424837906..1b0dce6832 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,10 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? o Minor bugfixes: - Fix a crash bug when trying to initialize the evdns module in Libevent 2. + - Stop logging at severity 'warn' when some other Tor client tries + to establish a circuit with us using weak DH keys. It's a protocol + violation, but that doesn't mean ordinary users need to hear about + it. Fixes the bug part of bug 1114. Bugfix on 0.1.0.13. Changes in version 0.2.2.5-alpha - 2009-10-11 From 56048637a5e34220d0a5c7abbef4eb56c9312f6c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 20:12:52 -0400 Subject: [PATCH 47/81] Only send the if_modified_since header for a v3 consensus. Spotted by xmux; bugfix on 0.2.0.10-alpha. (Bug introduced by 20b10859) --- ChangeLog | 8 ++++++++ src/or/directory.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index e8ec768408..2ea21b758e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +Changes in Version 0.2.1.21 - 20??-??-?? + o Minor bugfixes: + - Do not refuse to learn about authority certs and v2 networkstatus + documents that are older than the latest consensus. This bug might + have degraded client bootstrapping. Bugfix on 0.2.0.10-alpha. + Spotted and fixed by xmux. + + Changes in version 0.2.1.20 - 2009-10-15 o Major bugfixes: - Send circuit or stream sendme cells when our window has decreased diff --git a/src/or/directory.c b/src/or/directory.c index 5b8637a39d..8099e3376d 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -332,7 +332,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, return; } - if (DIR_PURPOSE_FETCH_CONSENSUS) { + if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { networkstatus_t *v = networkstatus_get_latest_consensus(); if (v) if_modified_since = v->valid_after + 180; From 8bada1ef67cd7e84f3f22f7e4ef8eb99a8252776 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 21:35:26 -0400 Subject: [PATCH 48/81] Add missing break statements for Coverity CIDs #406,407. The code for these was super-wrong, but will only break things when we reset an option on a platform where sizeof(time_t) is different from sizeof(int). --- src/or/config.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/or/config.c b/src/or/config.c index 4e2a1765dc..128aa4da34 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -2225,6 +2225,7 @@ option_clear(config_format_t *fmt, or_options_t *options, config_var_t *var) break; case CONFIG_TYPE_ISOTIME: *(time_t*)lvalue = 0; + break; case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_UINT: case CONFIG_TYPE_BOOL: @@ -2238,6 +2239,7 @@ option_clear(config_format_t *fmt, or_options_t *options, config_var_t *var) routerset_free(*(routerset_t**)lvalue); *(routerset_t**)lvalue = NULL; } + break; case CONFIG_TYPE_CSV: if (*(smartlist_t**)lvalue) { SMARTLIST_FOREACH(*(smartlist_t **)lvalue, char *, cp, tor_free(cp)); From 5c73da7faad1537f63c0f9923a25f3dda9df0de1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 22:12:40 -0400 Subject: [PATCH 49/81] Fix two memory leaks found by Coverity (CIDs 417-418) The first happens on an error case when a controller wants an impossible directory object. The second happens when we can't write our fingerprint file. --- src/or/control.c | 1 + src/or/router.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/or/control.c b/src/or/control.c index 5688b8e71f..f0178d74a9 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1536,6 +1536,7 @@ getinfo_helper_dir(control_connection_t *control_conn, if (res) { log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg); smartlist_free(descs); + tor_free(url); return -1; } SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, diff --git a/src/or/router.c b/src/or/router.c index fcfbe79112..dab4cb8938 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -588,6 +588,7 @@ init_keys(void) if (write_str_to_file(keydir, fingerprint_line, 0)) { log_err(LD_FS, "Error writing fingerprint line to file"); tor_free(keydir); + tor_free(cp); return -1; } } From 4256a964618ea48a34d900b6c77fc6d7f97138cb Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 26 Oct 2009 19:27:54 -0700 Subject: [PATCH 50/81] Fix bug 1066. If all authorities restart at once right before a consensus vote, nobody will vote about "Running", and clients will get a consensus with no usable relays. Instead, authorities refuse to build a consensus if this happens. --- ChangeLog | 4 ++++ src/or/dirvote.c | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 1b0dce6832..2e0633814a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,10 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? to establish a circuit with us using weak DH keys. It's a protocol violation, but that doesn't mean ordinary users need to hear about it. Fixes the bug part of bug 1114. Bugfix on 0.1.0.13. + - If all authorities restart at once right before a consensus vote, + nobody will vote about "Running", and clients will get a consensus + with no usable relays. Instead, authorities refuse to build a + consensus if this happens. Bugfix on 0.2.0.10-alpha; fixes bug 1066. Changes in version 0.2.2.5-alpha - 2009-10-11 diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 5ce3fd2ca7..65d7c477eb 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -2304,7 +2304,7 @@ static int dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ - int n_votes, n_voters; + int n_votes, n_voters, n_vote_running = 0; smartlist_t *votes = NULL, *votestrings = NULL; char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; @@ -2324,6 +2324,19 @@ dirvote_compute_consensuses(void) "%d of %d", n_votes, n_voters/2); goto err; } + tor_assert(pending_vote_list); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { + if (smartlist_string_isin(v->vote->known_flags, "Running")) + n_vote_running++; + }); + if (!n_vote_running) { + /* See task 1066. */ + log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating " + "and publishing a consensus without Running nodes " + "would make many clients stop working. Not " + "generating a consensus!"); + goto err; + } if (!(my_cert = get_my_v3_authority_cert())) { log_warn(LD_DIR, "Can't generate consensus without a certificate."); From e50e7395567c437370223a1cd20f29cd10bf3812 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 22:39:42 -0400 Subject: [PATCH 51/81] Add changelog to describe coverity fixes for 0.2.1.21 --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2ea21b758e..7023ea3bbe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,9 @@ Changes in Version 0.2.1.21 - 20??-??-?? documents that are older than the latest consensus. This bug might have degraded client bootstrapping. Bugfix on 0.2.0.10-alpha. Spotted and fixed by xmux. + - Fix a couple of very-hard-to-trigger memory leaks, and one hard-to- + trigger platform-specific option misparsing case found by Coverity + Scan. Changes in version 0.2.1.20 - 2009-10-15 From a457cd91fafdce6e7d63ef2089233768498335f2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 21:42:15 -0400 Subject: [PATCH 52/81] Clarification to suppress Coverity CID 405. Every or conn has an outbuf, but coverity has no way of knowing that. Add an assert to ease its conscience. --- src/or/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/or/main.c b/src/or/main.c index 9605b2bca8..537abcc832 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -731,6 +731,7 @@ run_connection_housekeeping(int i, time_t now) return; /* we're all done here, the rest is just for OR conns */ or_conn = TO_OR_CONN(conn); + tor_assert(conn->outbuf); if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: From cec698d29ed51ed219f907e12538e6945e535e4a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 21:52:35 -0400 Subject: [PATCH 53/81] Fix CID 409: check return value of base64_encode in tests --- src/test/test_crypto.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 670fd0048f..656c3c94fc 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -455,6 +455,7 @@ test_crypto_formats(void) strlcat(data1, " 2nd string that contains 35 chars.", 1024); i = base64_encode(data2, 1024, data1, 71); + test_assert(i >= 0); j = base64_decode(data3, 1024, data2, i); test_eq(j, 71); test_streq(data3, data1); From 5e4d53d535a3cc9903250b3df0caa829f1c5e4bf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 21:59:34 -0400 Subject: [PATCH 54/81] Remove checks for array existence. (CID 410..415) In C, the code "char x[10]; if (x) {...}" always takes the true branch of the if statement. Coverity notices this now. In some cases, we were testing arrays to make sure that an operation we wanted to do would suceed. Those cases are now always-true. In some cases, we were testing arrays to see if something was _set_. Those caes are now tests for strlen(s), or tests for !tor_mem_is_zero(d,len). --- src/common/crypto.c | 3 --- src/or/circuitbuild.c | 5 ++--- src/or/connection.c | 2 +- src/or/control.c | 2 +- src/or/rendcommon.c | 3 ++- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/common/crypto.c b/src/common/crypto.c index 5a39a1ab0c..4c880f6b6f 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1260,9 +1260,6 @@ crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key) tor_assert(env); tor_assert(key); - if (!env->key) - return -1; - memcpy(env->key, key, CIPHER_KEY_LEN); return 0; } diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 40c3a6b87f..91fa9d8db5 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1015,8 +1015,7 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) router_get_verbose_nickname(elt, ri); } else if ((rs = router_get_consensus_status_by_id(id))) { routerstatus_get_verbose_nickname(elt, rs); - } else if (hop->extend_info->nickname && - is_legal_nickname(hop->extend_info->nickname)) { + } else if (is_legal_nickname(hop->extend_info->nickname)) { elt[0] = '$'; base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN); elt[HEX_DIGEST_LEN+1]= '~'; @@ -1228,7 +1227,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) if (!n_conn) { /* not currently connected in a useful way. */ - const char *name = firsthop->extend_info->nickname ? + const char *name = strlen(firsthop->extend_info->nickname) ? firsthop->extend_info->nickname : fmt_addr(&firsthop->extend_info->addr); log_info(LD_CIRC, "Next router is %s: %s ", safe_str(name), msg?msg:"???"); circ->_base.n_hop = extend_info_dup(firsthop->extend_info); diff --git a/src/or/connection.c b/src/or/connection.c index ca71373f00..aca9b8b116 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -566,7 +566,7 @@ connection_about_to_close_connection(connection_t *conn) rep_hist_note_disconnect(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); - } else if (or_conn->identity_digest) { + } else if (!tor_digest_is_zero(or_conn->identity_digest)) { rep_hist_note_connection_died(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); diff --git a/src/or/control.c b/src/or/control.c index 0f744f7b96..d4b0bdb866 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -2263,7 +2263,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, char* exit_digest; if (circ->build_state && circ->build_state->chosen_exit && - circ->build_state->chosen_exit->identity_digest) { + !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { exit_digest = circ->build_state->chosen_exit->identity_digest; r = router_get_by_digest(exit_digest); } diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index c7eb2a9d08..9055f981bb 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -1245,7 +1245,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Decode/decrypt introduction points. */ if (intro_content) { if (rend_query->auth_type != REND_NO_AUTH && - rend_query->descriptor_cookie) { + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, From 98cd8c5d167450733fcef804838919e528e13ad3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 22:03:44 -0400 Subject: [PATCH 55/81] Fix a very stupid coverity complaint (CID 416). In its zeal to keep me from saying memset(x, '0', sizeof(x)), Coverity disallows memset(x, 48, sizeof(x)). Fine. I'll choose a different magic number, see if I care! --- src/test/test_dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index ca55c115ac..b7ee403299 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -837,7 +837,7 @@ test_dir_v3_networkstatus(void) rs->published_on = now-1000; strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); memset(rs->identity_digest, 34, DIGEST_LEN); - memset(rs->descriptor_digest, 48, DIGEST_LEN); + memset(rs->descriptor_digest, 47, DIGEST_LEN); rs->addr = 0xC0000203; rs->or_port = 500; rs->dir_port = 1999; From caa141617f324138b329051520dcdf0d9289d618 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 22:15:13 -0400 Subject: [PATCH 56/81] Fix dead code found by Coverity (CID 419). This was left over from an early draft of the microdescriptor code; it began to populate the signatures array of a networkstatus vote, even though there's no actual need to do that for a vote. --- src/or/dirserv.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 6ce6a461e1..d9a5decd6e 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -2609,11 +2609,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); voter->sigs = smartlist_create(); - { - document_signature_t *sig = tor_malloc_zero(sizeof(document_signature_t)); - memcpy(sig->identity_digest, identity_digest, DIGEST_LEN); - memcpy(sig->signing_key_digest, signing_key_digest, DIGEST_LEN); - } voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; From 134ac8059b18551e37f5b298f5a6362f07fa3ea7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 22:18:05 -0400 Subject: [PATCH 57/81] Fix the very noisy unit test memory leak of CID 420-421. On any failing case in test_util_config_line, we would leak a couple of strings. --- src/test/test_util.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 64f733581b..51b788e725 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -88,7 +88,7 @@ static void test_util_config_line(void) { char buf[1024]; - char *k, *v; + char *k=NULL, *v=NULL; const char *str; /* Test parse_config_line_from_str */ @@ -161,7 +161,8 @@ test_util_config_line(void) tor_free(k); tor_free(v); test_streq(str, ""); done: - ; + tor_free(k); + tor_free(v); } /** Test basic string functionality. */ From 385853a282138a614bb2bd8d6fe95a1aa573c944 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 22:35:29 -0400 Subject: [PATCH 58/81] Fix/annotate deadcode for CID 402,403 --- src/or/control.c | 44 ++++++++++++++++++++++---------------------- src/or/hibernate.c | 3 +++ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/or/control.c b/src/or/control.c index d4b0bdb866..7bc8fa7ae0 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -830,36 +830,36 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, retval = options_init_from_string(body, CMD_RUN_TOR, NULL, &errstring); - if (retval != SETOPT_OK) { + if (retval != SETOPT_OK) log_warn(LD_CONTROL, "Controller gave us config file that didn't validate: %s", errstring); - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - tor_fragile_assert(); - break; - } + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { if (*errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); - tor_free(errstring); - return 0; + } else { + send_control_done(conn); } - send_control_done(conn); return 0; } diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 89f9aa701b..d68682d730 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -182,6 +182,9 @@ accounting_parse_options(or_options_t *options, int validate_only) case UNIT_DAY: d = 0; break; + /* Coverity dislikes unreachable default cases; some compilers warn on + * switch statements missing a case. Tell Coverity not to worry. */ + /* coverity[dead_error_begin] */ default: tor_assert(0); } From 698aaeb1787c18723407ebcfdd4172826f0ded5b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 22:38:10 -0400 Subject: [PATCH 59/81] Note coverity fixes in changelog. --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 99afc39072..c5fc5f2ab8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,7 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? documents that are older than the latest consensus. This bug might have degraded client bootstrapping. Bugfix on 0.2.0.10-alpha. Spotted and fixed by xmux. + - Fix numerous small code-flaws found by Coverity Scan Rung 3. Changes in version 0.2.2.5-alpha - 2009-10-11 From 311315e077d193e680e0e7de98e28a9d10bcb002 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 23:13:29 -0400 Subject: [PATCH 60/81] Fix an accidentally removed free in 385853a282138a61, and repair a check. --- src/or/control.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/or/control.c b/src/or/control.c index 7bc8fa7ae0..009994302e 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -853,13 +853,14 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, break; } if (msg) { - if (*errstring) + if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); } else { send_control_done(conn); } + tor_free(errstring); return 0; } From 54973a45a693cf3e0dada2572016fa6695a51e75 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Oct 2009 23:14:53 -0400 Subject: [PATCH 61/81] Fix an apparently bogus check; fortunately, it seems to be untriggered. --- src/or/control.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/control.c b/src/or/control.c index f0178d74a9..ad9081da68 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -929,7 +929,7 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, tor_fragile_assert(); break; } - if (*errstring) + if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); From 19ddee5582bf6dc3d53cb31944de23277341eab6 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 26 Oct 2009 22:49:43 -0700 Subject: [PATCH 62/81] Fix bug 1042. If your relay can't keep up with the number of incoming create cells, it would log one warning per failure into your logs. Limit warnings to 1 per minute. --- ChangeLog | 3 +++ src/or/onion.c | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index f2d282fda4..5eb74f7efb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -37,6 +37,9 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? nobody will vote about "Running", and clients will get a consensus with no usable relays. Instead, authorities refuse to build a consensus if this happens. Bugfix on 0.2.0.10-alpha; fixes bug 1066. + - If your relay can't keep up with the number of incoming create + cells, it would log one warning per failure into your logs. Limit + warnings to 1 per minute. Bugfix on 0.0.2pre10; fixes bug 1042. Changes in version 0.2.2.5-alpha - 2009-10-11 diff --git a/src/or/onion.c b/src/or/onion.c index 58a51aedfe..f8913cd23b 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -58,11 +58,17 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) tor_assert(!ol_tail->next); if (ol_length >= get_options()->MaxOnionsPending) { - log_warn(LD_GENERAL, - "Your computer is too slow to handle this many circuit " - "creation requests! Please consider using the " - "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); +#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) + static time_t last_warned = 0; + time_t now = time(NULL); + if (last_warned + WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL < now) { + log_warn(LD_GENERAL, + "Your computer is too slow to handle this many circuit " + "creation requests! Please consider using the " + "MaxAdvertisedBandwidth config option or choosing a more " + "restricted exit policy."); + last_warned = now; + } tor_free(tmp); return -1; } From c8b27a8e9eed58ea1a8e8be36289b474f4942c11 Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Mon, 26 Oct 2009 23:09:10 -0700 Subject: [PATCH 63/81] Improve log statement when publishing v2 hs desc. --- src/or/rendservice.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/or/rendservice.c b/src/or/rendservice.c index f00cfd44d8..b6981d6258 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1553,6 +1553,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char *hs_dir_ip; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_digest_isin(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -1574,15 +1575,18 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, strlen(desc->desc_str), 0); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); + hs_dir_ip = tor_dup_ip(hs_dir->addr); log_info(LD_REND, "Sending publish request for v2 descriptor for " "service '%s' with descriptor ID '%s' with validity " "of %d seconds to hidden service directory '%s' on " - "port %d.", + "%s:%d.", safe_str(service_id), safe_str(desc_id_base32), seconds_valid, hs_dir->nickname, - hs_dir->dir_port); + hs_dir_ip, + hs_dir->or_port); + tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_digest_isin(successful_uploads, hs_dir->identity_digest)) smartlist_add(successful_uploads, hs_dir->identity_digest); From 56c2385157ee3fac81bb3f0c44fd933e0063ccde Mon Sep 17 00:00:00 2001 From: Karsten Loesing Date: Tue, 27 Oct 2009 01:03:41 -0700 Subject: [PATCH 64/81] Fix bug 1113. Bridges do not use the default exit policy, but reject *:* by default. --- ChangeLog | 2 ++ src/or/or.h | 3 ++- src/or/policies.c | 15 ++++++++++----- src/or/router.c | 2 +- src/test/test.c | 6 +++--- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5eb74f7efb..6d9535f03e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -40,6 +40,8 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? - If your relay can't keep up with the number of incoming create cells, it would log one warning per failure into your logs. Limit warnings to 1 per minute. Bugfix on 0.0.2pre10; fixes bug 1042. + - Bridges do not use the default exit policy, but reject *:* by + default. Fixes bug 1113. Changes in version 0.2.2.5-alpha - 2009-10-11 diff --git a/src/or/or.h b/src/or/or.h index 2fa4a797ad..bf415d8393 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -4353,7 +4353,8 @@ addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, uint16_t port, const smartlist_t *policy); int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address); + int rejectprivate, const char *local_address, + int add_default_policy); void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); int exit_policy_is_general_exit(smartlist_t *policy); int policy_is_reject_star(const smartlist_t *policy); diff --git a/src/or/policies.c b/src/or/policies.c index d55e86c184..023cd472f2 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -344,7 +344,8 @@ validate_addr_policies(or_options_t *options, char **msg) *msg = NULL; if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->ExitPolicyRejectPrivate, NULL)) + options->ExitPolicyRejectPrivate, NULL, + !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); /* The rest of these calls *append* to addr_policy. So don't actually @@ -829,14 +830,16 @@ exit_policy_remove_redundancies(smartlist_t *dest) "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" /** Parse the exit policy cfg into the linked list *dest. If - * cfg doesn't end in an absolute accept or reject, add the default exit + * cfg doesn't end in an absolute accept or reject and if + * add_default_policy is true, add the default exit * policy afterwards. If rejectprivate is true, prepend * "reject private:*" to the policy. Return -1 if we can't parse cfg, * else return 0. */ int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address) + int rejectprivate, const char *local_address, + int add_default_policy) { if (rejectprivate) { append_exit_policy_string(dest, "reject private:*"); @@ -848,8 +851,10 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, } if (parse_addr_policy(cfg, dest, -1)) return -1; - append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); - + if (add_default_policy) + append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); + else + append_exit_policy_string(dest, "reject *:*"); exit_policy_remove_redundancies(*dest); return 0; diff --git a/src/or/router.c b/src/or/router.c index 145301cd0c..2f5a9fd80b 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1312,7 +1312,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->ExitPolicyRejectPrivate, - ri->address); + ri->address, !options->BridgeRelay); if (desc_routerinfo) { /* inherit values */ ri->is_valid = desc_routerinfo->is_valid; diff --git a/src/test/test.c b/src/test/test.c index 839d9469eb..d85f1f0f87 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -629,7 +629,7 @@ test_policy_summary_helper(const char *policy_str, line.value = (char *)policy_str; line.next = NULL; - r = policies_parse_exit_policy(&line, &policy, 0, NULL); + r = policies_parse_exit_policy(&line, &policy, 0, NULL, 1); test_eq(r, 0); summary = policy_summarize(policy); @@ -675,7 +675,7 @@ test_policies(void) compare_addr_to_addr_policy(0xc0a80102, 2, policy)); policy2 = NULL; - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL)); + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL, 1)); test_assert(policy2); test_assert(!exit_policy_is_general_exit(policy)); @@ -699,7 +699,7 @@ test_policies(void) line.key = (char*)"foo"; line.value = (char*)"accept *:80,reject private:*,reject *:*"; line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL)); + test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL, 1)); test_assert(policy); //test_streq(policy->string, "accept *:80"); //test_streq(policy->next->string, "reject *:*"); From 2aac39a779c125dbc0cc510d0c306e9df83b33c4 Mon Sep 17 00:00:00 2001 From: Jacob Appelbaum Date: Wed, 21 Oct 2009 21:21:57 -0700 Subject: [PATCH 65/81] Implement DisableAllSwap to avoid putting secret info in page files. This commit implements a new config option: 'DisableAllSwap' This option probably only works properly when Tor is started as root. We added two new functions: tor_mlockall() and tor_set_max_memlock(). tor_mlockall() attempts to mlock() all current and all future memory pages. For tor_mlockall() to work properly we set the process rlimits for memory to RLIM_INFINITY (and beyond) inside of tor_set_max_memlock(). We behave differently from mlockall() by only allowing tor_mlockall() to be called one single time. All other calls will result in a return code of 1. It is not possible to change DisableAllSwap while running. A sample configuration item was added to the torrc.complete.in config file. A new item in the man page for DisableAllSwap was added. Thanks to Moxie Marlinspike and Chris Palmer for their feedback on this patch. Please note that we make no guarantees about the quality of your OS and its mlock/mlockall implementation. It is possible that this will do nothing at all. It is also possible that you can ulimit the mlock properties of a given user such that root is not required. This has not been extensively tested and is unsupported. I have included some comments for possible ways we can handle this on win32. --- ChangeLog | 10 ++++ doc/tor.1.in | 13 +++++ src/common/compat.c | 99 ++++++++++++++++++++++++++++++++++++ src/common/compat.h | 2 + src/config/torrc.complete.in | 3 ++ src/or/config.c | 18 +++++++ src/or/or.h | 3 ++ 7 files changed, 148 insertions(+) diff --git a/ChangeLog b/ChangeLog index 6d9535f03e..0109ff5054 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,16 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? algorithms for signatures and resource selection. Newer formats are signed with SHA256, with a possibility for moving to a better hash algorithm in the future. + - New DisableAllSwap option. If set to 1, Tor will attempt to lock all + current and future memory pages. On supported platforms, this should + effectively disable any and all attempts to page out memory. Under the + hood, DisableAllSwap uses mlockall() on unix-like platforms. Windows is + currently unsupported. We believe that this feature works on modern + Gnu/Linux distributions. Mac OS X appears to be broken by design. On + reasonable *BSD systems it should also be supported but this is untested. + This option requires that you start your Tor as root. If you use + DisableAllSwap, please consider using the User option to properly reduce + the privileges of your Tor. o Code simplifications and refactorings: - Numerous changes, bugfixes, and workarounds from Nathan Freitas diff --git a/doc/tor.1.in b/doc/tor.1.in index 739b889fb5..1a71026aad 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -234,6 +234,19 @@ the default hidden service authorities, but not the directory or bridge authorities. .LP .TP +\fBDisableAllSwap \fR\fB0\fR|\fB1\fR\fP +If set to 1, Tor will attempt to lock all current and future memory pages. +On supported platforms, this should effectively disable any and all attempts +to page out memory. Under the hood, DisableAllSwap uses mlockall() on unix-like +platforms. Windows is currently unsupported. We believe that this feature works +on modern Gnu/Linux distributions. Mac OS X appears to be broken by design. On +reasonable *BSD systems it should also be supported but this is untested. This +option requires that you start your Tor as root. If you use DisableAllSwap, +please consider using the User option to properly reduce the privileges of +your Tor. +(Default: 0) +.LP +.TP \fBFetchDirInfoEarly \fR\fB0\fR|\fB1\fR\fP If set to 1, Tor will always fetch directory information like other directory caches, even if you don't meet the normal criteria for diff --git a/src/common/compat.c b/src/common/compat.c index e1a275de11..9e5dca3525 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -2204,6 +2204,105 @@ tor_threads_init(void) } #endif +#ifdef HAVE_SYS_MMAN_H +/** Attempt to raise the current and max rlimit to infinity for our process. + * This only needs to be done once and can probably only be done when we have + * not already dropped privileges. + */ +static int +tor_set_max_memlock(void) +{ + /* Future consideration for Windows is probably SetProcessWorkingSetSize + * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK + * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx + */ + + struct rlimit limit; + int ret; + + /* Do we want to report current limits first? This is not really needed. */ + ret = getrlimit(RLIMIT_MEMLOCK, &limit); + if (ret == -1) { + log_warn(LD_GENERAL, "Could not get RLIMIT_MEMLOCK: %s", strerror(errno)); + return -1; + } + + /* RLIM_INFINITY is -1 on some platforms. */ + limit.rlim_cur = RLIM_INFINITY; + limit.rlim_max = RLIM_INFINITY; + + ret = setrlimit(RLIMIT_MEMLOCK, &limit); + if (ret == -1) { + if (errno == EPERM) { + log_warn(LD_GENERAL, "You appear to lack permissions to change memory " + "limits. Are you root?"); + log_warn(LD_GENERAL, "Unable to raise RLIMIT_MEMLOCK: %s", + strerror(errno)); + } else { + log_warn(LD_GENERAL, "Could not raise RLIMIT_MEMLOCK: %s", + strerror(errno)); + } + return -1; + } + + return 0; +} +#endif + +/** Attempt to lock all current and all future memory pages. + * This should only be called once and while we're privileged. + * Like mlockall() we return 0 when we're successful and -1 when we're not. + * Unlike mlockall() we return 1 if we've already attempted to lock memory. + */ +int +tor_mlockall(void) +{ + static int memory_lock_attempted = 0; + int ret; + + if (memory_lock_attempted) { + return 1; + } + + memory_lock_attempted = 1; + + /* + * Future consideration for Windows may be VirtualLock + * VirtualLock appears to implement mlock() but not mlockall() + * + * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx + */ + +#ifdef HAVE_SYS_MMAN_H + ret = tor_set_max_memlock(); + if (ret == 0) { + /* Perhaps we only want to log this if we're in a verbose mode? */ + log_notice(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY."); + } + + ret = mlockall(MCL_CURRENT|MCL_FUTURE); + if (ret == 0) { + log_notice(LD_GENERAL, "Insecure OS paging is effectively disabled."); + return 0; + } else { + if (errno == ENOSYS) { + /* Apple - it's 2009! I'm looking at you. Grrr. */ + log_notice(LD_GENERAL, "It appears that mlockall() is not available on " + "your platform."); + } else if (errno == EPERM) { + log_notice(LD_GENERAL, "It appears that you lack the permissions to " + "lock memory. Are you root?"); + } + log_notice(LD_GENERAL, "Unable to lock all current and future memory " + "pages: %s", strerror(errno)); + return -1; + } +#else + log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); + return -1; +#endif +} + /** Identity of the "main" thread */ static unsigned long main_thread_id = -1; diff --git a/src/common/compat.h b/src/common/compat.h index edd09d8683..554ae8919f 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -509,6 +509,8 @@ typedef struct tor_mutex_t { #endif } tor_mutex_t; +int tor_mlockall(void); + #ifdef TOR_IS_MULTITHREADED tor_mutex_t *tor_mutex_new(void); void tor_mutex_init(tor_mutex_t *m); diff --git a/src/config/torrc.complete.in b/src/config/torrc.complete.in index 2fbf494e56..6dbec2fbf9 100644 --- a/src/config/torrc.complete.in +++ b/src/config/torrc.complete.in @@ -79,6 +79,9 @@ #DirServer moria2 v1 18.244.0.114:80 719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF #DirServer tor26 v1 86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D +## Attempt to lock current and future memory pages and effectively disable swap +# DisableAllSwap 0|1 + ## On startup, setgid to this user. #Group GID diff --git a/src/or/config.c b/src/or/config.c index 5a0ced29d5..b6a52a85de 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -195,6 +195,7 @@ static config_var_t _option_vars[] = { OBSOLETE("DirRecordUsageSaveInterval"), V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), + V(DisableAllSwap, BOOL, "0"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), @@ -456,6 +457,8 @@ static config_var_description_t options_description[] = { { "DirServer", "Tor only trusts directories signed with one of these " "servers' keys. Used to override the standard list of directory " "authorities." }, + { "DisableAllSwap", "Tor will attempt a simple memory lock that " + "will prevent leaking of all information in memory to the swap file." }, /* { "FastFirstHopPK", "" }, */ /* FetchServerDescriptors, FetchHidServDescriptors, * FetchUselessDescriptors */ @@ -1115,6 +1118,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #endif + /* Attempt to lock all current and future memory with mlockall() only once */ + if (options->DisableAllSwap) { + if (tor_mlockall() == -1) { + *msg = tor_strdup("DisableAllSwap failure. Do you have proper " + "permissions?"); + goto done; + } + } + /* Setuid/setgid as appropriate */ if (options->User) { if (switch_id(options->User) != 0) { @@ -3834,6 +3846,12 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } + if (old->DisableAllSwap != new_val->DisableAllSwap) { + *msg = tor_strdup("While Tor is running, changing DisableAllSwap " + "is not allowed."); + return -1; + } + return 0; } diff --git a/src/or/or.h b/src/or/or.h index bf415d8393..767ad95720 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2287,6 +2287,9 @@ typedef struct { * stop building circuits? */ int StrictEntryNodes; /**< Boolean: When none of our EntryNodes are up, do we * stop building circuits? */ + int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our + * process for all current and future memory. */ + routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs * not to use in circuits. */ From eb1faf8a0a51592618b5595fc152d26d71e3f43e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 4 Nov 2009 11:38:57 -0500 Subject: [PATCH 66/81] Fix a URL in a log message. --- src/or/directory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/or/directory.c b/src/or/directory.c index e703bd438a..1d3c43ec0c 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1689,8 +1689,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) (int) body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".", + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); From ce0a89e2624471272ffc4950c5069d9b81a7f0b9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 5 Nov 2009 18:13:08 -0500 Subject: [PATCH 67/81] Make Tor work with OpenSSL 0.9.8l To fix a major security problem related to incorrect use of SSL/TLS renegotiation, OpenSSL has turned off renegotiation by default. We are not affected by this security problem, however, since we do renegotiation right. (Specifically, we never treat a renegotiated credential as authenticating previous communication.) Nevertheless, OpenSSL's new behavior requires us to explicitly turn renegotiation back on in order to get our protocol working again. Amusingly, this is not so simple as "set the flag when you create the SSL object" , since calling connect or accept seems to clear the flags. For belt-and-suspenders purposes, we clear the flag once the Tor handshake is done. There's no way to exploit a second handshake either, but we might as well not allow it. --- ChangeLog | 6 ++++++ src/common/tortls.c | 34 ++++++++++++++++++++++++++++++++++ src/common/tortls.h | 1 + src/or/connection_or.c | 2 ++ 4 files changed, 43 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7023ea3bbe..4799e74078 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ Changes in Version 0.2.1.21 - 20??-??-?? + o Major bugfixes: + - Work around a security feature in OpenSSL 0.9.8l that prevents our + handshake from working unless we explicitly tell OpenSSL that we are + using SSL renegotiation safely. We are, of course, but OpenSSL + 0.9.8l won't work unless we say we are. + o Minor bugfixes: - Do not refuse to learn about authority certs and v2 networkstatus documents that are older than the latest consensus. This bug might diff --git a/src/common/tortls.c b/src/common/tortls.c index c6b11e9a6e..bcc6780a65 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -154,6 +154,7 @@ static X509* tor_tls_create_certificate(crypto_pk_env_t *rsa, const char *cname, const char *cname_sign, unsigned int lifetime); +static void tor_tls_unblock_renegotiation(tor_tls_t *tls); /** Global tls context. We keep it here because nobody else needs to * touch it. */ @@ -904,6 +905,36 @@ tor_tls_set_renegotiate_callback(tor_tls_t *tls, #endif } +/** If this version of openssl requires it, turn on renegotiation on + * tls. (Our protocol never requires this for security, but it's nice + * to use belt-and-suspenders here.) + */ +static void +tor_tls_unblock_renegotiation(tor_tls_t *tls) +{ +#ifdef SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION + /* Yes, we know what we are doing here. No, we do not treat a renegotiation + * as authenticating any earlier-received data. */ + tls->ssl->s3->flags |= SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION; +#else + (void)tls; +#endif +} + +/** If this version of openssl supports it, turn off renegotiation on + * tls. (Our protocol never requires this for security, but it's nice + * to use belt-and-suspenders here.) + */ +void +tor_tls_block_renegotiation(tor_tls_t *tls) +{ +#ifdef SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION + tls->ssl->s3->flags &= ~SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION; +#else + (void)tls; +#endif +} + /** Return whether this tls initiated the connect (client) or * received it (server). */ int @@ -1026,6 +1057,9 @@ tor_tls_handshake(tor_tls_t *tls) } else { r = SSL_connect(tls->ssl); } + /* We need to call this here and not earlier, since OpenSSL has a penchant + * for clearing its flags when you say accept or connect. */ + tor_tls_unblock_renegotiation(tls); r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO); if (ERR_peek_error() != 0) { tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, diff --git a/src/common/tortls.h b/src/common/tortls.h index d00690911c..871fec3365 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -65,6 +65,7 @@ int tor_tls_read(tor_tls_t *tls, char *cp, size_t len); int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n); int tor_tls_handshake(tor_tls_t *tls); int tor_tls_renegotiate(tor_tls_t *tls); +void tor_tls_block_renegotiation(tor_tls_t *tls); int tor_tls_shutdown(tor_tls_t *tls); int tor_tls_get_pending_bytes(tor_tls_t *tls); size_t tor_tls_get_forced_write_size(tor_tls_t *tls); diff --git a/src/or/connection_or.c b/src/or/connection_or.c index b4e80926be..2a52b3fcd6 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -844,6 +844,7 @@ connection_or_tls_renegotiated_cb(tor_tls_t *tls, void *_conn) /* Don't invoke this again. */ tor_tls_set_renegotiate_callback(tls, NULL, NULL); + tor_tls_block_renegotiation(tls); if (connection_tls_finish_handshake(conn) < 0) { /* XXXX_TLS double-check that it's ok to do this from inside read. */ @@ -1087,6 +1088,7 @@ connection_tls_finish_handshake(or_connection_t *conn) connection_or_init_conn_from_address(conn, &conn->_base.addr, conn->_base.port, digest_rcvd, 0); } + tor_tls_block_renegotiation(conn->tls); return connection_or_set_state_open(conn); } else { conn->_base.state = OR_CONN_STATE_OR_HANDSHAKING; From 2db0256372e02edfa517ad5b56106f7210ddd7ab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 6 Nov 2009 15:25:41 -0500 Subject: [PATCH 68/81] Add changelog entry to 0.2.2.x about openssl 0.9.8l fix --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 679d576ea3..3edcbae646 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,12 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? DisableAllSwap, please consider using the User option to properly reduce the privileges of your Tor. + o Major bugfixes: + - Work around a security feature in OpenSSL 0.9.8l that prevents our + handshake from working unless we explicitly tell OpenSSL that we are + using SSL renegotiation safely. We are, of course, but OpenSSL + 0.9.8l won't work unless we say we are. + o Code simplifications and refactorings: - Numerous changes, bugfixes, and workarounds from Nathan Freitas to help Tor build correctly for Android phones. From 69c0147ea6725a63f254333867c0504528c62daf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 8 Nov 2009 00:38:46 -0500 Subject: [PATCH 69/81] Fix building from a separate build directory. --- src/test/Makefile.am | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/Makefile.am b/src/test/Makefile.am index ea7c67eda7..cdb5d85c28 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -4,9 +4,8 @@ noinst_PROGRAMS = test AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ - -DBINDIR="\"$(bindir)\"" - -AM_CFLAGS = -I../or + -DBINDIR="\"$(bindir)\"" \ + -I"$(top_srcdir)/src/or" # -L flags need to go in LDFLAGS. -l flags need to go in LDADD. # This seems to matter nowhere but on windows, but I assure you that it From 22f674fcb88411d8b56a3423fb0c09bb7745529c Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 12 Nov 2009 01:31:26 -0500 Subject: [PATCH 70/81] Fix a memory leak on directory authorities during voting Fix a memory leak on directory authorities during voting that was introduced in 0.2.2.1-alpha. Found via valgrind. --- ChangeLog | 2 ++ src/or/dirvote.c | 1 + 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 3edcbae646..dd05eb51d2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -58,6 +58,8 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? warnings to 1 per minute. Bugfix on 0.0.2pre10; fixes bug 1042. - Bridges do not use the default exit policy, but reject *:* by default. Fixes bug 1113. + - Fix a memory leak on directory authorities during voting that was + introduced in 0.2.2.1-alpha. Found via valgrind. Changes in version 0.2.2.5-alpha - 2009-10-11 diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 65d7c477eb..5de5da1985 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1305,6 +1305,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(versions); smartlist_free(exitsummaries); tor_free(bandwidths); + tor_free(measured_bws); } /* Add a signature. */ From 751e9b2bb64f5c4f5af6e3c9105c85deae17943e Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Fri, 13 Nov 2009 19:57:10 +0100 Subject: [PATCH 71/81] New upstream version --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 00eae0f0ac..90afa018bb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +tor (0.2.1.20-1) unstable; urgency=low + + * New upstream version. + + -- Peter Palfrader Fri, 13 Nov 2009 19:02:47 +0100 + tor (0.2.1.19-1) unstable; urgency=low * New upstream version. From 6f1fe7e941b8f6d62829852059862b00beed5a6e Mon Sep 17 00:00:00 2001 From: Jacob Appelbaum Date: Sat, 14 Nov 2009 16:43:22 -0500 Subject: [PATCH 72/81] Fix compilation with with bionic libc. This fixes bug 1147: bionic doesn't have an actual implementation of mlockall(); mlockall() is merely in the headers but not actually in the library. This prevents Tor compilation with the bionic libc for Android handsets. --- configure.in | 5 +++++ src/common/compat.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 7ffa578d69..32f28b5661 100644 --- a/configure.in +++ b/configure.in @@ -628,6 +628,11 @@ if test x$tcmalloc = xyes ; then LDFLAGS="-ltcmalloc $LDFLAGS" fi +# By default, we're going to assume we don't have mlockall() +# bionic and other platforms have various broken mlockall subsystems +# some of systems don't have a working mlockall, some aren't linkable +AC_CHECK_FUNCS(mlockall) + # Allow user to specify an alternate syslog facility AC_ARG_WITH(syslog-facility, [ --with-syslog-facility=LOG syslog facility to use (default=LOG_DAEMON)], diff --git a/src/common/compat.c b/src/common/compat.c index 9e5dca3525..96012e2e01 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -2273,7 +2273,7 @@ tor_mlockall(void) * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx */ -#ifdef HAVE_SYS_MMAN_H +#ifdef HAVE_MLOCKALL ret = tor_set_max_memlock(); if (ret == 0) { /* Perhaps we only want to log this if we're in a verbose mode? */ From 2bcb90a308859126489bf63e3d2598901fcefcfc Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 17 Nov 2009 07:02:17 -0500 Subject: [PATCH 73/81] clean up changelog for the 0.2.2.6-alpha release --- ChangeLog | 65 +++++++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/ChangeLog b/ChangeLog index dd05eb51d2..d3fd6c0c43 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,52 +1,45 @@ -Changes in version 0.2.2.6-alpha - 2009-10-?? +Changes in version 0.2.2.6-alpha - 2009-11-17 o Major features: - - Directory authorities can now create, vote, and serve on multiple + - Directory authorities can now create, vote on, and serve multiple parallel formats of directory data as part of their voting process. - This is a partial implementation of Proposal 162: "Publish the - consensus in multiple flavors." - - Directory authorities can now agree on and publish small summaries of - router information that clients can use in place of regular server - descriptors. This will eventually allow clients to use far less - bandwidth for downloading information about the network. This begins - the implementation of of Proposal 158: "Clients download a consensus + - Microdescriptors". + Partially implements Proposal 162: "Publish the consensus in + multiple flavors". + - Directory authorities can now agree on and publish small summaries + of router information that clients can use in place of regular + server descriptors. This transition will eventually allow clients + to use far less bandwidth for downloading information about the + network. Begins the implementation of Proposal 158: "Clients + download consensus + microdescriptors". - The directory voting system is now extensible to use multiple hash - algorithms for signatures and resource selection. Newer formats are - signed with SHA256, with a possibility for moving to a better hash - algorithm in the future. + algorithms for signatures and resource selection. Newer formats + are signed with SHA256, with a possibility for moving to a better + hash algorithm in the future. - New DisableAllSwap option. If set to 1, Tor will attempt to lock all - current and future memory pages. On supported platforms, this should - effectively disable any and all attempts to page out memory. Under the - hood, DisableAllSwap uses mlockall() on unix-like platforms. Windows is - currently unsupported. We believe that this feature works on modern - Gnu/Linux distributions. Mac OS X appears to be broken by design. On - reasonable *BSD systems it should also be supported but this is untested. - This option requires that you start your Tor as root. If you use - DisableAllSwap, please consider using the User option to properly reduce - the privileges of your Tor. + current and future memory pages via mlockall(). On supported + platforms (modern Linux and probably BSD but not Windows or OS X), + this should effectively disable any and all attempts to page out + memory. This option requires that you start your Tor as root -- + if you use DisableAllSwap, please consider using the User option + to properly reduce the privileges of your Tor. + - Numerous changes, bugfixes, and workarounds from Nathan Freitas + to help Tor build correctly for Android phones. o Major bugfixes: - Work around a security feature in OpenSSL 0.9.8l that prevents our - handshake from working unless we explicitly tell OpenSSL that we are - using SSL renegotiation safely. We are, of course, but OpenSSL - 0.9.8l won't work unless we say we are. - - o Code simplifications and refactorings: - - Numerous changes, bugfixes, and workarounds from Nathan Freitas - to help Tor build correctly for Android phones. - - Begun converting Tor's signature and message digest logic to handle - multiple hash algorithms. + handshake from working unless we explicitly tell OpenSSL that we + are using SSL renegotiation safely. We are, but OpenSSL 0.9.8l + won't work unless we say we are. o Minor bugfixes: - Fix a crash bug when trying to initialize the evdns module in - Libevent 2. + Libevent 2. Bugfix on 0.2.1.16-rc. - Stop logging at severity 'warn' when some other Tor client tries to establish a circuit with us using weak DH keys. It's a protocol violation, but that doesn't mean ordinary users need to hear about it. Fixes the bug part of bug 1114. Bugfix on 0.1.0.13. - Do not refuse to learn about authority certs and v2 networkstatus - documents that are older than the latest consensus. This bug might - have degraded client bootstrapping. Bugfix on 0.2.0.10-alpha. + documents that are older than the latest consensus. This bug might + have degraded client bootstrapping. Bugfix on 0.2.0.10-alpha. Spotted and fixed by xmux. - Fix numerous small code-flaws found by Coverity Scan Rung 3. - If all authorities restart at once right before a consensus vote, @@ -56,8 +49,8 @@ Changes in version 0.2.2.6-alpha - 2009-10-?? - If your relay can't keep up with the number of incoming create cells, it would log one warning per failure into your logs. Limit warnings to 1 per minute. Bugfix on 0.0.2pre10; fixes bug 1042. - - Bridges do not use the default exit policy, but reject *:* by - default. Fixes bug 1113. + - Bridges now use "reject *:*" as their default exit policy. Bugfix + on 0.2.0.3-alpha; fixes bug 1113. - Fix a memory leak on directory authorities during voting that was introduced in 0.2.2.1-alpha. Found via valgrind. From 2ebd22152eeb5a9dcfc1a8c508ebd57a836290e1 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 17 Nov 2009 07:39:15 -0500 Subject: [PATCH 74/81] only complain when rejecting a descriptor if it has contact info --- src/or/dirserv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index d9a5decd6e..3700cd134e 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -523,7 +523,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); time_t now; - int severity = complain ? LOG_NOTICE : LOG_INFO; + int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ From e722ffa60508d4959566eb210aa8e2a00c6cf899 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 17 Nov 2009 14:24:59 -0500 Subject: [PATCH 75/81] Do not report a partially-successful detached signature add as failed. Also, regenerate the detached-signature document whenever any signatures are successfully added. --- src/or/dirvote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 5de5da1985..448c1270cd 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -2568,7 +2568,7 @@ dirvote_add_signatures_to_all_pending_consensuses( n_added += res; } - if (errors) { + if (errors && !n_added) { r = -1; goto err; } From 0656c12b077f1241ca9b3b471b3d5f1b34dead94 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Tue, 17 Nov 2009 15:35:14 -0500 Subject: [PATCH 76/81] add the 0.2.1.20 changelog blurb, plus update the releasenotes --- ChangeLog | 12 +++++++++ ReleaseNotes | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4799e74078..64910dd341 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,18 @@ Changes in Version 0.2.1.21 - 20??-??-?? Changes in version 0.2.1.20 - 2009-10-15 + Tor 0.2.1.20 fixes a crash bug when you're accessing many hidden + services at once, prepares for more performance improvements, and + fixes a bunch of smaller bugs. + + The Windows and OS X bundles also include a more recent Vidalia, + and switch from Privoxy to Polipo. + + The OS X installers are now drag and drop. It's best to un-install + Tor/Vidalia and then install this new bundle, rather than upgrade. If + you want to upgrade, you'll need to update the paths for Tor and Polipo + in the Vidalia Settings window. + o Major bugfixes: - Send circuit or stream sendme cells when our window has decreased by 100 cells, not when it has decreased by 101 cells. Bug uncovered diff --git a/ReleaseNotes b/ReleaseNotes index bd2e3090a3..a93a56d377 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -3,6 +3,81 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.2.1.20 - 2009-10-15 + Tor 0.2.1.20 fixes a crash bug when you're accessing many hidden + services at once, prepares for more performance improvements, and + fixes a bunch of smaller bugs. + + The Windows and OS X bundles also include a more recent Vidalia, + and switch from Privoxy to Polipo. + + The OS X installers are now drag and drop. It's best to un-install + Tor/Vidalia and then install this new bundle, rather than upgrade. If + you want to upgrade, you'll need to update the paths for Tor and Polipo + in the Vidalia Settings window. + + o Major bugfixes: + - Send circuit or stream sendme cells when our window has decreased + by 100 cells, not when it has decreased by 101 cells. Bug uncovered + by Karsten when testing the "reduce circuit window" performance + patch. Bugfix on the 54th commit on Tor -- from July 2002, + before the release of Tor 0.0.0. This is the new winner of the + oldest-bug prize. + - Fix a remotely triggerable memory leak when a consensus document + contains more than one signature from the same voter. Bugfix on + 0.2.0.3-alpha. + - Avoid segfault in rare cases when finishing an introduction circuit + as a client and finding out that we don't have an introduction key + for it. Fixes bug 1073. Reported by Aaron Swartz. + + o Major features: + - Tor now reads the "circwindow" parameter out of the consensus, + and uses that value for its circuit package window rather than the + default of 1000 cells. Begins the implementation of proposal 168. + + o New directory authorities: + - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory + authority. + - Move moria1 and tonga to alternate IP addresses. + + o Minor bugfixes: + - Fix a signed/unsigned compile warning in 0.2.1.19. + - Fix possible segmentation fault on directory authorities. Bugfix on + 0.2.1.14-rc. + - Fix an extremely rare infinite recursion bug that could occur if + we tried to log a message after shutting down the log subsystem. + Found by Matt Edman. Bugfix on 0.2.0.16-alpha. + - Fix an obscure bug where hidden services on 64-bit big-endian + systems might mis-read the timestamp in v3 introduce cells, and + refuse to connect back to the client. Discovered by "rotor". + Bugfix on 0.2.1.6-alpha. + - We were triggering a CLOCK_SKEW controller status event whenever + we connect via the v2 connection protocol to any relay that has + a wrong clock. Instead, we should only inform the controller when + it's a trusted authority that claims our clock is wrong. Bugfix + on 0.2.0.20-rc; starts to fix bug 1074. Reported by SwissTorExit. + - We were telling the controller about CHECKING_REACHABILITY and + REACHABILITY_FAILED status events whenever we launch a testing + circuit or notice that one has failed. Instead, only tell the + controller when we want to inform the user of overall success or + overall failure. Bugfix on 0.1.2.6-alpha. Fixes bug 1075. Reported + by SwissTorExit. + - Don't warn when we're using a circuit that ends with a node + excluded in ExcludeExitNodes, but the circuit is not used to access + the outside world. This should help fix bug 1090. Bugfix on + 0.2.1.6-alpha. + - Work around a small memory leak in some versions of OpenSSL that + stopped the memory used by the hostname TLS extension from being + freed. + + o Minor features: + - Add a "getinfo status/accepted-server-descriptor" controller + command, which is the recommended way for controllers to learn + whether our server descriptor has been successfully received by at + least on directory authority. Un-recommend good-server-descriptor + getinfo and status events until we have a better design for them. + + Changes in version 0.2.1.19 - 2009-07-28 Tor 0.2.1.19 fixes a major bug with accessing and providing hidden services. From 9be682942c14112c5af63e866d33558c45d609c9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 18 Nov 2009 11:26:44 -0500 Subject: [PATCH 77/81] Not everybody likes debugging printfs as much as I --- src/or/dirvote.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 448c1270cd..f745db6fc4 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1417,26 +1417,22 @@ networkstatus_add_detached_signatures(networkstatus_t *target, /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { - puts("A"); *msg_out = "Valid-After times do not match " "when adding detached signatures to consensus"; return -1; } if (target->fresh_until != sigs->fresh_until) { - puts("B"); *msg_out = "Fresh-until times do not match " "when adding detached signatures to consensus"; return -1; } if (target->valid_until != sigs->valid_until) { - puts("C"); *msg_out = "Valid-until times do not match " "when adding detached signatures to consensus"; return -1; } siglist = strmap_get(sigs->signatures, flavor); if (!siglist) { - puts("D"); *msg_out = "No signatures for given consensus flavor"; return -1; } @@ -1447,7 +1443,6 @@ networkstatus_add_detached_signatures(networkstatus_t *target, int n_matches = 0; digest_algorithm_t alg; if (!digests) { - puts("D"); *msg_out = "No digests for given consensus flavor"; return -1; } @@ -1456,16 +1451,12 @@ networkstatus_add_detached_signatures(networkstatus_t *target, if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { ++n_matches; } else { - printf("F %d\n", alg); - printf("%s\n", hex_str(target->digests.d[alg], DIGEST256_LEN)); - printf("%s\n", hex_str(digests->d[alg], DIGEST256_LEN)); *msg_out = "Mismatched digest."; return -1; } } } if (!n_matches) { - puts("G"); *msg_out = "No regognized digests for given consensus flavor"; } } From b5462efd60381db9a96c14c6968984a0381e6c33 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 19 Nov 2009 14:11:54 -0500 Subject: [PATCH 78/81] remove the 0.2.1.20 debian changelog from master's changelog --- debian/changelog | 6 ------ 1 file changed, 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 35a022815e..45b6b910f0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -62,12 +62,6 @@ tor (0.2.2.1-alpha-1) experimental; urgency=low -- Peter Palfrader Thu, 03 Sep 2009 15:10:26 +0200 -tor (0.2.1.20-1) unstable; urgency=low - - * New upstream version. - - -- Peter Palfrader Fri, 13 Nov 2009 19:02:47 +0100 - tor (0.2.1.19-1) unstable; urgency=low * New upstream version. From 1ee580407ccb91304cf7ae24c0d809f57e6b4ccc Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Thu, 19 Nov 2009 14:16:11 -0500 Subject: [PATCH 79/81] bump to 0.2.2.6-alpha --- ChangeLog | 2 +- configure.in | 2 +- contrib/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8a3432844a..c00fcdb4e5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -Changes in version 0.2.2.6-alpha - 2009-11-17 +Changes in version 0.2.2.6-alpha - 2009-11-19 o Major features: - Directory authorities can now create, vote on, and serve multiple parallel formats of directory data as part of their voting process. diff --git a/configure.in b/configure.in index 32f28b5661..6f2baf7e01 100644 --- a/configure.in +++ b/configure.in @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.2.5-alpha-dev) +AM_INIT_AUTOMAKE(tor, 0.2.2.6-alpha) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index 44660edf00..5cf9f2245c 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -9,7 +9,7 @@ !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.2.5-alpha-dev" +!define VERSION "0.2.2.6-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 37a1b10264..101fd0cad0 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -226,7 +226,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.2.5-alpha-dev" +#define VERSION "0.2.2.6-alpha" From c9935358b0f47b2896c68d0fe63260138adebd63 Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Mon, 23 Nov 2009 18:58:42 +0100 Subject: [PATCH 80/81] New upstream version --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 45b6b910f0..17364f4650 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +tor (0.2.2.6-alpha-1) experimental; urgency=low + + * New upstream version. + + -- Peter Palfrader Mon, 23 Nov 2009 18:52:04 +0100 + tor (0.2.2.5-alpha-1) experimental; urgency=low * New upstream version. From 50feca0dcbf9934f2bbe6fa4b84ea76d06890100 Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Mon, 23 Nov 2009 18:59:05 +0100 Subject: [PATCH 81/81] Drop debian/patches/0a58567c-work-with-reneg-ssl.dpatch (part of upstream) --- debian/changelog | 2 + debian/patches/00list | 1 - .../0a58567c-work-with-reneg-ssl.dpatch | 129 ------------------ 3 files changed, 2 insertions(+), 130 deletions(-) delete mode 100644 debian/patches/0a58567c-work-with-reneg-ssl.dpatch diff --git a/debian/changelog b/debian/changelog index 17364f4650..11b2dc3815 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ tor (0.2.2.6-alpha-1) experimental; urgency=low * New upstream version. + - Drop debian/patches/0a58567c-work-with-reneg-ssl.dpatch + (part of upstream). -- Peter Palfrader Mon, 23 Nov 2009 18:52:04 +0100 diff --git a/debian/patches/00list b/debian/patches/00list index 63c52165c8..9260bb7ed5 100644 --- a/debian/patches/00list +++ b/debian/patches/00list @@ -3,4 +3,3 @@ 06_add_compile_time_defaults.dpatch 07_log_to_file_by_default.dpatch 14_fix_geoip_warning -0a58567c-work-with-reneg-ssl.dpatch diff --git a/debian/patches/0a58567c-work-with-reneg-ssl.dpatch b/debian/patches/0a58567c-work-with-reneg-ssl.dpatch deleted file mode 100644 index 886e2d18ae..0000000000 --- a/debian/patches/0a58567c-work-with-reneg-ssl.dpatch +++ /dev/null @@ -1,129 +0,0 @@ -#! /bin/sh -e - -if [ $# -lt 1 ]; then - echo "`basename $0`: script expects -patch|-unpatch as argument" >&2 - exit 1 -fi - -[ -f debian/patches/00patch-opts ] && . debian/patches/00patch-opts -patch_opts="${patch_opts:--f --no-backup-if-mismatch} ${2:+-d $2}" - -case "$1" in - -patch) patch -p1 ${patch_opts} < $0;; - -unpatch) patch -R -p1 ${patch_opts} < $0;; - *) - echo "`basename $0`: script expects -patch|-unpatch as argument" >&2 - exit 1;; -esac - -exit 0 - -#diff --git a/ChangeLog b/ChangeLog -#index 0109ff5..679d576 100644 -#--- a/ChangeLog -#+++ b/ChangeLog -#@@ -311,6 +311,12 @@ Changes in version 0.2.2.1-alpha - 2009-08-26 -# -# -# Changes in Version 0.2.1.21 - 20??-??-?? -#+ o Major bugfixes: -#+ - Work around a security feature in OpenSSL 0.9.8l that prevents our -#+ handshake from working unless we explicitly tell OpenSSL that we are -#+ using SSL renegotiation safely. We are, of course, but OpenSSL -#+ 0.9.8l won't work unless we say we are. -#+ -# o Minor bugfixes: -# - Do not refuse to learn about authority certs and v2 networkstatus -# documents that are older than the latest consensus. This bug might -@DPATCH@ -diff --git a/src/common/tortls.c b/src/common/tortls.c -index 6e09325..ff49ecf 100644 ---- a/src/common/tortls.c -+++ b/src/common/tortls.c -@@ -154,6 +154,7 @@ static X509* tor_tls_create_certificate(crypto_pk_env_t *rsa, - const char *cname, - const char *cname_sign, - unsigned int lifetime); -+static void tor_tls_unblock_renegotiation(tor_tls_t *tls); - - /** Global tls context. We keep it here because nobody else needs to - * touch it. */ -@@ -927,6 +928,36 @@ tor_tls_set_renegotiate_callback(tor_tls_t *tls, - #endif - } - -+/** If this version of openssl requires it, turn on renegotiation on -+ * tls. (Our protocol never requires this for security, but it's nice -+ * to use belt-and-suspenders here.) -+ */ -+static void -+tor_tls_unblock_renegotiation(tor_tls_t *tls) -+{ -+#ifdef SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION -+ /* Yes, we know what we are doing here. No, we do not treat a renegotiation -+ * as authenticating any earlier-received data. */ -+ tls->ssl->s3->flags |= SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION; -+#else -+ (void)tls; -+#endif -+} -+ -+/** If this version of openssl supports it, turn off renegotiation on -+ * tls. (Our protocol never requires this for security, but it's nice -+ * to use belt-and-suspenders here.) -+ */ -+void -+tor_tls_block_renegotiation(tor_tls_t *tls) -+{ -+#ifdef SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION -+ tls->ssl->s3->flags &= ~SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION; -+#else -+ (void)tls; -+#endif -+} -+ - /** Return whether this tls initiated the connect (client) or - * received it (server). */ - int -@@ -1058,6 +1089,9 @@ tor_tls_handshake(tor_tls_t *tls) - if (oldstate != tls->ssl->state) - log_debug(LD_HANDSHAKE, "After call, %p was in state %s", - tls, ssl_state_to_string(tls->ssl->state)); -+ /* We need to call this here and not earlier, since OpenSSL has a penchant -+ * for clearing its flags when you say accept or connect. */ -+ tor_tls_unblock_renegotiation(tls); - r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO, LD_HANDSHAKE); - if (ERR_peek_error() != 0) { - tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, LD_HANDSHAKE, -diff --git a/src/common/tortls.h b/src/common/tortls.h -index d006909..871fec3 100644 ---- a/src/common/tortls.h -+++ b/src/common/tortls.h -@@ -65,6 +65,7 @@ int tor_tls_read(tor_tls_t *tls, char *cp, size_t len); - int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n); - int tor_tls_handshake(tor_tls_t *tls); - int tor_tls_renegotiate(tor_tls_t *tls); -+void tor_tls_block_renegotiation(tor_tls_t *tls); - int tor_tls_shutdown(tor_tls_t *tls); - int tor_tls_get_pending_bytes(tor_tls_t *tls); - size_t tor_tls_get_forced_write_size(tor_tls_t *tls); -diff --git a/src/or/connection_or.c b/src/or/connection_or.c -index c3d35e1..bbd6439 100644 ---- a/src/or/connection_or.c -+++ b/src/or/connection_or.c -@@ -799,6 +799,7 @@ connection_or_tls_renegotiated_cb(tor_tls_t *tls, void *_conn) - - /* Don't invoke this again. */ - tor_tls_set_renegotiate_callback(tls, NULL, NULL); -+ tor_tls_block_renegotiation(tls); - - if (connection_tls_finish_handshake(conn) < 0) { - /* XXXX_TLS double-check that it's ok to do this from inside read. */ -@@ -1045,6 +1046,7 @@ connection_tls_finish_handshake(or_connection_t *conn) - connection_or_init_conn_from_address(conn, &conn->_base.addr, - conn->_base.port, digest_rcvd, 0); - } -+ tor_tls_block_renegotiation(conn->tls); - return connection_or_set_state_open(conn); - } else { - conn->_base.state = OR_CONN_STATE_OR_HANDSHAKING;