Give tor_cert_get_id_digests() fail-fast behavior

Right now we can take the digests only of an RSA key, and only expect to
take the digests of an RSA key.  The old tor_cert_get_id_digests() would
return a good set of digests for an RSA key, and an all-zero one for a
non-RSA key.  This behavior is too error-prone: it carries the risk that
we will someday check two non-RSA keys for equality and conclude that
they must be equal because they both have the same (zero) "digest".

Instead, let's have tor_cert_get_id_digests() return NULL for keys we
can't handle, and make its callers explicitly test for NULL.

src/common/tortls.c

@@ -102,6 +102,7 @@ struct tor_cert_t {
   X509 *cert;
   uint8_t *encoded;
   size_t encoded_len;
+  unsigned pkey_digests_set : 1;
   digests_t cert_digests;
   digests_t pkey_digests;
 };
@@ -734,6 +735,7 @@ tor_cert_new(X509 *x509_cert)
       (rsa = EVP_PKEY_get1_RSA(pkey))) {
     crypto_pk_env_t *pk = _crypto_new_pk_env_rsa(rsa);
     crypto_pk_get_all_digests(pk, &cert->pkey_digests);
+    cert->pkey_digests_set = 1;
     crypto_free_pk_env(pk);
     EVP_PKEY_free(pkey);
   }
@@ -794,11 +796,15 @@ tor_cert_get_der(const tor_cert_t *cert,
   *size_out = cert->encoded_len;
 }
 
-/** Return a set of digests for the public key in <b>cert</b>. */
+/** Return a set of digests for the public key in <b>cert</b>, or NULL if this
+ * cert's public key is not one we know how to take the digest of. */
 const digests_t *
 tor_cert_get_id_digests(const tor_cert_t *cert)
 {
-  return &cert->pkey_digests;
+  if (cert->pkey_digests_set)
+    return &cert->pkey_digests;
+  else
+    return NULL;
 }
 
 /** Return a set of digests for the public key in <b>cert</b>. */

src/or/command.c

@@ -939,8 +939,12 @@ command_process_cert_cell(var_cell_t *cell, or_connection_t *conn)
 
     conn->handshake_state->authenticated = 1;
     {
-      crypto_pk_env_t *identity_rcvd = tor_tls_cert_get_key(id_cert);
       const digests_t *id_digests = tor_cert_get_id_digests(id_cert);
+      crypto_pk_env_t *identity_rcvd;
+      if (!id_digests)
+        ERR("Couldn't compute digests for key in ID cert");
+
+      identity_rcvd = tor_tls_cert_get_key(id_cert);
       memcpy(conn->handshake_state->authenticated_peer_id,
              id_digests->d[DIGEST_SHA1], DIGEST_LEN);
       connection_or_set_circid_type(conn, identity_rcvd);
@@ -1172,6 +1176,9 @@ command_process_authenticate_cell(var_cell_t *cell, or_connection_t *conn)
     const digests_t *id_digests =
       tor_cert_get_id_digests(conn->handshake_state->id_cert);
 
+    /* This must exist; we checked key type when reading the cert. */
+    tor_assert(id_digests);
+
     memcpy(conn->handshake_state->authenticated_peer_id,
            id_digests->d[DIGEST_SHA1], DIGEST_LEN);
 

src/or/connection_or.c

@@ -2060,12 +2060,17 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
 
   {
     const tor_cert_t *id_cert=NULL, *link_cert=NULL;
+    const digests_t *my_digests, *their_digests;
     const uint8_t *my_id, *their_id, *client_id, *server_id;
     if (tor_tls_get_my_certs(0, &link_cert, &id_cert))
       return -1;
-    my_id = (uint8_t*)tor_cert_get_id_digests(id_cert)->d[DIGEST_SHA256];
+    my_digests = tor_cert_get_id_digests(id_cert);
-    their_id = (uint8_t*)
+    their_digests = tor_cert_get_id_digests(conn->handshake_state->id_cert);
-      tor_cert_get_id_digests(conn->handshake_state->id_cert)->d[DIGEST_SHA256];
+    tor_assert(my_digests);
+    tor_assert(their_digests);
+    my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
+    their_id = (uint8_t*)their_digests->d[DIGEST_SHA256];
+
     client_id = server ? their_id : my_id;
     server_id = server ? my_id : their_id;