Code to download, parse, and store microdesc consensuses

changes/microdesc_dl

@@ -0,0 +1,4 @@
+  o Major features:
+    - Caches now download and cache all the consensus flavors that
+      they know about.  This allows them to assess which microdescriptors
+      they need to fetch.

src/or/directory.c

@@ -361,9 +361,24 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
   }
 
   if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    networkstatus_t *v = networkstatus_get_latest_consensus();
-    if (v)
-      if_modified_since = v->valid_after + 180;
+    int flav = FLAV_NS;
+    networkstatus_t *v;
+    if (resource)
+      flav = networkstatus_parse_flavor_name(resource);
+
+    if (flav != -1) {
+      /* IF we have a parsed consensus of this type, we can do an
+       * if-modified-time based on it. */
+      v = networkstatus_get_latest_consensus_by_flavor(flav);
+      if (v)
+        if_modified_since = v->valid_after + 180;
+    } else {
+      /* Otherwise it might be a consensus we don't parse, but which we
+       * do cache.  Look at the cached copy, perhaps. */
+      cached_dir_t *cd = dirserv_get_consensus(resource ? resource : "ns");
+      if (cd)
+        if_modified_since = cd->published + 180;
+    }
   }
 
   if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
@@ -598,7 +613,9 @@ connection_dir_request_failed(dir_connection_t *conn)
       connection_dir_bridge_routerdesc_failed(conn);
     connection_dir_download_routerdesc_failed(conn);
   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    networkstatus_consensus_download_failed(0);
+    const char *flavname =
+      conn->requested_resource ? conn->requested_resource : "ns";
+    networkstatus_consensus_download_failed(0, flavname);
   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
     log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
              conn->_base.address);
@@ -955,12 +972,16 @@ _compare_strs(const void **a, const void **b)
  * This url depends on whether or not the server we go to
  * is sufficiently new to support conditional consensus downloading,
  * i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
+ *
+ * If 'resource' is provided, it is the name of a consensus flavor to request.
  */
 static char *
-directory_get_consensus_url(int supports_conditional_consensus)
+directory_get_consensus_url(int supports_conditional_consensus,
+                            const char *resource)
 {
-  char *url;
-  size_t len;
+  char *url = NULL;
+  const char *hyphen = resource ? "-" : "";
+  const char *flavor = resource ? resource : "";
 
   if (supports_conditional_consensus) {
     char *authority_id_list;
@@ -982,16 +1003,15 @@ directory_get_consensus_url(int supports_conditional_consensus)
     authority_id_list = smartlist_join_strings(authority_digests,
                                                "+", 0, NULL);
 
-    len = strlen(authority_id_list)+64;
-    url = tor_malloc(len);
-    tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
-                 authority_id_list);
+    tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
+                 hyphen, flavor, authority_id_list);
 
     SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
     smartlist_free(authority_digests);
     tor_free(authority_id_list);
   } else {
-    url = tor_strdup("/tor/status-vote/current/consensus.z");
+    tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s.z",
+                 hyphen, flavor);
   }
   return url;
 }
@@ -1072,10 +1092,11 @@ directory_send_command(dir_connection_t *conn,
       tor_snprintf(url, len, "/tor/status/%s", resource);
       break;
     case DIR_PURPOSE_FETCH_CONSENSUS:
-      tor_assert(!resource);
+      /* resource is optional.  If present, it's a flavor name */
       tor_assert(!payload);
       httpcommand = "GET";
-      url = directory_get_consensus_url(supports_conditional_consensus);
+      url = directory_get_consensus_url(supports_conditional_consensus,
+                                        resource);
       log_info(LD_DIR, "Downloading consensus from %s using %s",
                hoststring, url);
       break;
@@ -1690,6 +1711,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
 
   if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
     int r;
+    const char *flavname =
+      conn->requested_resource ? conn->requested_resource : "ns";
     if (status_code != 200) {
       int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
       log(severity, LD_DIR,
@@ -1698,18 +1721,18 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
            status_code, escaped(reason), conn->_base.address,
            conn->_base.port);
       tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(status_code);
+      networkstatus_consensus_download_failed(status_code, flavname);
       return -1;
     }
     log_info(LD_DIR,"Received consensus directory (size %d) from server "
              "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
-    if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) {
+    if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) {
       log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
-             "Unable to load consensus directory downloaded from "
+             "Unable to load %s consensus directory downloaded from "
              "server '%s:%d'. I'll try again soon.",
-             conn->_base.address, conn->_base.port);
+             flavname, conn->_base.address, conn->_base.port);
       tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(0);
+      networkstatus_consensus_download_failed(0, flavname);
       return -1;
     }
     /* launches router downloads as needed */

src/or/dirserv.c

@@ -1263,7 +1263,8 @@ static cached_dir_t cached_runningrouters;
  * cached_dir_t. */
 static digestmap_t *cached_v2_networkstatus = NULL;
 
-/** Map from flavor name to the v3 consensuses that we're currently serving. */
+/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
+ * currently serving. */
 static strmap_t *cached_consensuses = NULL;
 
 /** Possibly replace the contents of <b>d</b> with the value of

src/or/networkstatus.c

@@ -44,8 +44,19 @@ static strmap_t *named_server_map = NULL;
  * as unnamed for some server in the consensus. */
 static strmap_t *unnamed_server_map = NULL;
 
-/** Most recently received and validated v3 consensus network status. */
-static networkstatus_t *current_consensus = NULL;
+/** Most recently received and validated v3 consensus network status,
+ * of whichever type we are using for our own circuits.  This will be the same
+ * as one of current_ns_consensus or current_md_consensus.
+ */
+#define current_consensus current_ns_consensus
+
+/** Most recently received and validated v3 "ns"-flavored consensus network
+ * status. */
+static networkstatus_t *current_ns_consensus = NULL;
+
+/** Most recently received and validated v3 "microdec"-flavored consensus
+ * network status. */
+static networkstatus_t *current_md_consensus = NULL;
 
 /** A v3 consensus networkstatus that we've received, but which we don't
  * have enough certificates to be happy about. */
@@ -1199,6 +1210,24 @@ update_v2_networkstatus_cache_downloads(time_t now)
   }
 }
 
+static int
+we_want_to_fetch_flavor(or_options_t *options, int flavor)
+{
+  if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
+    /* This flavor is crazy; we don't want it */
+    /*XXXX handle unrecognized flavors later */
+    return 0;
+  }
+  if (authdir_mode_v3(options) || directory_caches_dir_info(options)) {
+    /* We want to serve all flavors to others, regardless if we would use
+     * it ourselves. */
+    return 1;
+  }
+  /* Otherwise, we want the flavor only if we want to use it to build
+   * circuits. */
+  return (flavor == USABLE_CONSENSUS_FLAVOR);
+}
+
 /** How many times will we try to fetch a consensus before we give up? */
 #define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8
 /** How long will we hang onto a possibly live consensus for which we're
@@ -1211,48 +1240,65 @@ static void
 update_consensus_networkstatus_downloads(time_t now)
 {
   int i;
+  or_options_t *options = get_options();
+
   if (!networkstatus_get_live_consensus(now))
     time_to_download_next_consensus = now; /* No live consensus? Get one now!*/
   if (time_to_download_next_consensus > now)
     return; /* Wait until the current consensus is older. */
-  /* XXXXNM Microdescs: may need to download more types. */
-  if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now,
-                                CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
-    return; /* We failed downloading a consensus too recently. */
-  if (connection_get_by_type_purpose(CONN_TYPE_DIR,
-                                     DIR_PURPOSE_FETCH_CONSENSUS))
-    return; /* There's an in-progress download.*/
 
   for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
-    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    /* XXXX need some way to download unknown flavors if we are caching. */
+    const char *resource;
+    consensus_waiting_for_certs_t *waiting;
+
+    if (! we_want_to_fetch_flavor(options, i))
+      continue;
+
+    resource = i==FLAV_NS ? NULL : networkstatus_get_flavor_name(i);
+
+    if (!download_status_is_ready(&consensus_dl_status[i], now,
+                                  CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
+      continue; /* We failed downloading a consensus too recently. */
+    if (connection_dir_get_by_purpose_and_resource(
+                                DIR_PURPOSE_FETCH_CONSENSUS, resource))
+      continue; /* There's an in-progress download.*/
+
+    waiting = &consensus_waiting_for_certs[i];
     if (waiting->consensus) {
       /* XXXX make sure this doesn't delay sane downloads. */
-      if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now)
-        return; /* We're still getting certs for this one. */
-      else {
+      if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) {
+        continue; /* We're still getting certs for this one. */
+      } else {
         if (!waiting->dl_failed) {
-          download_status_failed(&consensus_dl_status[FLAV_NS], 0);
+          download_status_failed(&consensus_dl_status[i], 0);
           waiting->dl_failed=1;
         }
       }
     }
-  }
 
-  log_info(LD_DIR, "Launching networkstatus consensus download.");
-  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
-                               ROUTER_PURPOSE_GENERAL, NULL,
-                               PDS_RETRY_IF_NO_SERVERS);
+    log_info(LD_DIR, "Launching %s networkstatus consensus download.",
+             networkstatus_get_flavor_name(i));
+
+    directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+                                 ROUTER_PURPOSE_GENERAL, resource,
+                                 PDS_RETRY_IF_NO_SERVERS);
+  }
 }
 
 /** Called when an attempt to download a consensus fails: note that the
  * failure occurred, and possibly retry. */
 void
-networkstatus_consensus_download_failed(int status_code)
+networkstatus_consensus_download_failed(int status_code, const char *flavname)
 {
-  /* XXXXNM Microdescs: may need to handle more types. */
-  download_status_failed(&consensus_dl_status[FLAV_NS], status_code);
-  /* Retry immediately, if appropriate. */
-  update_consensus_networkstatus_downloads(time(NULL));
+  int flav = networkstatus_parse_flavor_name(flavname);
+  if (flav >= 0) {
+    tor_assert(flav < N_CONSENSUS_FLAVORS);
+    /* XXXX handle unrecognized flavors */
+    download_status_failed(&consensus_dl_status[flav], status_code);
+    /* Retry immediately, if appropriate. */
+    update_consensus_networkstatus_downloads(time(NULL));
+  }
 }
 
 /** How long do we (as a cache) wait after a consensus becomes non-fresh
@@ -1373,7 +1419,10 @@ update_certificate_downloads(time_t now)
                                     now);
   }
 
-  authority_certs_fetch_missing(current_consensus, now);
+  if (current_ns_consensus)
+    authority_certs_fetch_missing(current_ns_consensus, now);
+  if (current_ns_consensus)
+    authority_certs_fetch_missing(current_md_consensus, now);
 }
 
 /** Return 1 if we have a consensus but we don't have enough certificates
@@ -1405,6 +1454,18 @@ networkstatus_get_latest_consensus(void)
   return current_consensus;
 }
 
+/** DOCDOC */
+networkstatus_t *
+networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  if (f == FLAV_NS)
+    return current_ns_consensus;
+  else if (f == FLAV_MICRODESC)
+    return current_md_consensus;
+  else
+    tor_assert(0);
+}
+
 /** Return the most recent consensus that we have downloaded, or NULL if it is
  * no longer live. */
 networkstatus_t *
@@ -1569,6 +1630,7 @@ networkstatus_set_current_consensus(const char *consensus,
   const digests_t *current_digests = NULL;
   consensus_waiting_for_certs_t *waiting = NULL;
   time_t current_valid_after = 0;
+  int free_consensus = 1;
 
   if (flav < 0) {
     /* XXXX we don't handle unrecognized flavors yet. */
@@ -1614,9 +1676,16 @@ networkstatus_set_current_consensus(const char *consensus,
   if (!strcmp(flavor, "ns")) {
     consensus_fname = get_datadir_fname("cached-consensus");
     unverified_fname = get_datadir_fname("unverified-consensus");
-    if (current_consensus) {
-      current_digests = &current_consensus->digests;
-      current_valid_after = current_consensus->valid_after;
+    if (current_ns_consensus) {
+      current_digests = &current_ns_consensus->digests;
+      current_valid_after = current_ns_consensus->valid_after;
+    }
+  } else if (!strcmp(flavor, "microdesc")) {
+    consensus_fname = get_datadir_fname("cached-microdesc-consensus");
+    unverified_fname = get_datadir_fname("unverified-microdesc-consensus");
+    if (current_md_consensus) {
+      current_digests = &current_md_consensus->digests;
+      current_valid_after = current_md_consensus->valid_after;
     }
   } else {
     cached_dir_t *cur;
@@ -1702,11 +1771,21 @@ networkstatus_set_current_consensus(const char *consensus,
 
   if (flav == USABLE_CONSENSUS_FLAVOR) {
     notify_control_networkstatus_changed(current_consensus, c);
-
-    if (current_consensus) {
-      networkstatus_copy_old_consensus_info(c, current_consensus);
-      networkstatus_vote_free(current_consensus);
+  }
+  if (flav == FLAV_NS) {
+    if (current_ns_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+      networkstatus_vote_free(current_ns_consensus);
     }
+    current_ns_consensus = c;
+    free_consensus = 0; /* avoid free */
+  } else if (flav == FLAV_MICRODESC) {
+    if (current_md_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_md_consensus);
+      networkstatus_vote_free(current_md_consensus);
+    }
+    current_md_consensus = c;
+    free_consensus = 0; /* avoid free */
   }
 
   waiting = &consensus_waiting_for_certs[flav];
@@ -1739,11 +1818,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (flav == USABLE_CONSENSUS_FLAVOR) {
-    current_consensus = c;
-    c = NULL; /* Prevent free. */
-
     /* XXXXNM Microdescs: needs a non-ns variant. */
     update_consensus_networkstatus_fetch_time(now);
+
     dirvote_recalculate_timing(options, now);
     routerstatus_list_update_named_server_map();
     cell_ewma_set_scale_factor(options, current_consensus);
@@ -1758,11 +1835,11 @@ networkstatus_set_current_consensus(const char *consensus,
     write_str_to_file(consensus_fname, consensus, 0);
   }
 
-  if (ftime_definitely_before(now, current_consensus->valid_after)) {
+  if (ftime_definitely_before(now, c->valid_after)) {
     char tbuf[ISO_TIME_LEN+1];
     char dbuf[64];
-    long delta = now - current_consensus->valid_after;
-    format_iso_time(tbuf, current_consensus->valid_after);
+    long delta = now - c->valid_after;
+    format_iso_time(tbuf, c->valid_after);
     format_time_interval(dbuf, sizeof(dbuf), delta);
     log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
              "consensus network status document (%s GMT).  Tor needs an "
@@ -1776,7 +1853,8 @@ networkstatus_set_current_consensus(const char *consensus,
 
   result = 0;
  done:
-  networkstatus_vote_free(c);
+  if (free_consensus)
+    networkstatus_vote_free(c);
   tor_free(consensus_fname);
   tor_free(unverified_fname);
   return result;
@@ -2028,7 +2106,7 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
 void
 signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
 {
-  networkstatus_t *ns = current_consensus;
+  networkstatus_t *ns = current_ns_consensus;
   if (!ns)
     return;
 
@@ -2264,8 +2342,9 @@ networkstatus_free_all(void)
 
   digestmap_free(v2_download_status_map, _tor_free);
   v2_download_status_map = NULL;
-  networkstatus_vote_free(current_consensus);
-  current_consensus = NULL;
+  networkstatus_vote_free(current_ns_consensus);
+  networkstatus_vote_free(current_md_consensus);
+  current_md_consensus = current_ns_consensus = NULL;
 
   for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
     consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];

src/or/networkstatus.h

@@ -53,7 +53,8 @@ routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname,
                                                        int warn_if_unnamed);
 const char *networkstatus_get_router_digest_by_nickname(const char *nickname);
 int networkstatus_nickname_is_unnamed(const char *nickname);
-void networkstatus_consensus_download_failed(int status_code);
+void networkstatus_consensus_download_failed(int status_code,
+                                             const char *flavname);
 void update_consensus_networkstatus_fetch_time(time_t now);
 int should_delay_dir_fetches(or_options_t *options);
 void update_networkstatus_downloads(time_t now);
@@ -61,6 +62,8 @@ void update_certificate_downloads(time_t now);
 int consensus_is_waiting_for_certs(void);
 networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest);
 networkstatus_t *networkstatus_get_latest_consensus(void);
+networkstatus_t *networkstatus_get_latest_consensus_by_flavor(
+                                                  consensus_flavor_t f);
 networkstatus_t *networkstatus_get_live_consensus(time_t now);
 networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now);
 #define NSSET_FROM_CACHE 1