Merge branch 'tor-github/pr/891' into maint-0.4.0

changes/diagnostic_28223_redux

@@ -0,0 +1,4 @@
+  o Minor features (diagnostic):
+    - Add more diagnostic log messages in an attempt to solve
+      the issue of NUL bytes appearing in a microdescriptor cache.
+      Related to ticket 28223.

src/feature/dirparse/microdesc_parse.c

@@ -160,7 +160,22 @@ microdescs_parse_from_string(const char *s, const char *eos,
 
     if (tokenize_string(area, s, start_of_next_microdesc, tokens,
                         microdesc_token_table, flags)) {
-      log_warn(LD_DIR, "Unparseable microdescriptor");
+      const char *location;
+      switch (where) {
+        case SAVED_NOWHERE:
+          location = "download or generated string";
+          break;
+        case SAVED_IN_CACHE:
+          location = "cache";
+          break;
+        case SAVED_IN_JOURNAL:
+          location = "journal";
+          break;
+        default:
+          location = "unknown location";
+          break;
+      }
+      log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location);
       goto next;
     }
 

src/feature/nodelist/microdesc.c

@@ -70,6 +70,8 @@ struct microdesc_cache_t {
 };
 
 static microdesc_cache_t *get_microdesc_cache_noload(void);
+static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                              const char *activity);
 
 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
 static inline unsigned int
@@ -223,6 +225,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
   }
 
   md->off = tor_fd_getpos(fd);
+  warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
+                    "dumping a microdescriptor");
   written = write_all_to_fd(fd, md->body, md->bodylen);
   if (written != (ssize_t)md->bodylen) {
     written = written < 0 ? 0 : written;
@@ -482,6 +486,27 @@ microdesc_cache_clear(microdesc_cache_t *cache)
   cache->bytes_dropped = 0;
 }
 
+static void
+warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                  const char *activity)
+{
+  const char *nul_found = memchr(inp, 0, len);
+  if (BUG(nul_found)) {
+    log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
+             "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
+             activity, offset, (nul_found - inp), len);
+    const char *start_excerpt_at, *eos = inp + len;
+    if ((nul_found - inp) >= 16)
+      start_excerpt_at = nul_found - 16;
+    else
+      start_excerpt_at = inp;
+    size_t excerpt_len = MIN(32, eos - start_excerpt_at);
+    char tmp[65];
+    base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
+    log_warn(LD_BUG, "      surrounding string: %s", tmp);
+  }
+}
+
 /** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
  * for the first time.  Return 0 on success, -1 on failure. */
 int
@@ -499,6 +524,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
 
   mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
   if (mm) {
+    warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
     added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
                                     SAVED_IN_CACHE, 0, -1, NULL);
     if (added) {
@@ -511,6 +537,8 @@ microdesc_cache_reload(microdesc_cache_t *cache)
                                      RFTS_IGNORE_MISSING, &st);
   if (journal_content) {
     cache->journal_len = (size_t) st.st_size;
+    warn_if_nul_found(journal_content, cache->journal_len, 0,
+                      "reading microdesc journal");
     added = microdescs_add_to_cache(cache, journal_content,
                                     journal_content+st.st_size,
                                     SAVED_IN_JOURNAL, 0, -1, NULL);

src/lib/fdio/fdio.c

@@ -17,12 +17,16 @@
 #ifdef _WIN32
 #include <windows.h>
 #endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 
 #include "lib/fdio/fdio.h"
 #include "lib/cc/torint.h"
 #include "lib/err/torerr.h"
 
 #include <stdlib.h>
+#include <stdio.h>
 
 /** @{ */
 /** Some old versions of Unix didn't define constants for these values,