mirrors/tor
1
0
Fork 0
mirror of https://gitlab.torproject.org/tpo/core/tor.git synced 2025-02-25 07:07:52 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'storage_labeled_squashed'

Browse source
This commit is contained in:
Nick Mathewson 2017-04-06 11:49:00 -04:00
commit 9d34a1e052
18 changed files with 1534 additions and 360 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

320
src/common/confline.c Normal file
View file

@ -0,0 +1,320 @@
/* Copyright (c) 2001 Matej Pfajfar.
* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2017, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#include "compat.h"
#include "confline.h"
#include "torlog.h"
#include "util.h"
/** Helper: allocate a new configuration option mapping 'key' to 'val',
* append it to *<b>lst</b>. */
void
config_line_append(config_line_t **lst,
const char *key,
const char *val)
{
tor_assert(lst);
config_line_t *newline;
newline = tor_malloc_zero(sizeof(config_line_t));
newline->key = tor_strdup(key);
newline->value = tor_strdup(val);
newline->next = NULL;
while (*lst)
lst = &((*lst)->next);
(*lst) = newline;
}
/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or
* NULL if no such key exists.
*
* (In options parsing, this is for handling commandline-only options only;
* other options should be looked up in the appropriate data structure.) */
const config_line_t *
config_line_find(const config_line_t *lines,
const char *key)
{
const config_line_t *cl;
for (cl = lines; cl; cl = cl->next) {
if (!strcmp(cl->key, key))
return cl;
}
return NULL;
}
/** Helper: parse the config string and strdup into key/value
* strings. Set *result to the list, or NULL if parsing the string
* failed. Return 0 on success, -1 on failure. Warn and ignore any
* misformatted lines.
*
* If <b>extended</b> is set, then treat keys beginning with / and with + as
* indicating "clear" and "append" respectively. */
int
config_get_lines(const char *string, config_line_t **result, int extended)
{
config_line_t *list = NULL, **next;
char *k, *v;
const char *parse_err;
next = &list;
do {
k = v = NULL;
string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
if (!string) {
log_warn(LD_CONFIG, "Error while parsing configuration: %s",
parse_err?parse_err:"<unknown>");
config_free_lines(list);
tor_free(k);
tor_free(v);
return -1;
}
if (k && v) {
unsigned command = CONFIG_LINE_NORMAL;
if (extended) {
if (k[0] == '+') {
char *k_new = tor_strdup(k+1);
tor_free(k);
k = k_new;
command = CONFIG_LINE_APPEND;
} else if (k[0] == '/') {
char *k_new = tor_strdup(k+1);
tor_free(k);
k = k_new;
tor_free(v);
v = tor_strdup("");
command = CONFIG_LINE_CLEAR;
}
}
/* This list can get long, so we keep a pointer to the end of it
* rather than using config_line_append over and over and getting
* n^2 performance. */
*next = tor_malloc_zero(sizeof(config_line_t));
(*next)->key = k;
(*next)->value = v;
(*next)->next = NULL;
(*next)->command = command;
next = &((*next)->next);
} else {
tor_free(k);
tor_free(v);
}
} while (*string);
*result = list;
return 0;
}
/**
* Free all the configuration lines on the linked list <b>front</b>.
*/
void
config_free_lines(config_line_t *front)
{
config_line_t *tmp;
while (front) {
tmp = front;
front = tmp->next;
tor_free(tmp->key);
tor_free(tmp->value);
tor_free(tmp);
}
}
/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
config_line_t *
config_lines_dup(const config_line_t *inp)
{
return config_lines_dup_and_filter(inp, NULL);
}
/** Return a newly allocated deep copy of the lines in <b>inp</b>,
* but only the ones that match <b>key</b>. */
config_line_t *
config_lines_dup_and_filter(const config_line_t *inp,
const char *key)
{
config_line_t *result = NULL;
config_line_t **next_out = &result;
while (inp) {
if (key && strcasecmpstart(inp->key, key)) {
inp = inp->next;
continue;
}
*next_out = tor_malloc_zero(sizeof(config_line_t));
(*next_out)->key = tor_strdup(inp->key);
(*next_out)->value = tor_strdup(inp->value);
inp = inp->next;
next_out = &((*next_out)->next);
}
(*next_out) = NULL;
return result;
}
/** Return true iff a and b contain identical keys and values in identical
* order. */
int
config_lines_eq(config_line_t *a, config_line_t *b)
{
while (a && b) {
if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
return 0;
a = a->next;
b = b->next;
}
if (a || b)
return 0;
return 1;
}
/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
int
config_count_key(const config_line_t *a, const char *key)
{
int n = 0;
while (a) {
if (!strcasecmp(a->key, key)) {
++n;
}
a = a->next;
}
return n;
}
/** Given a string containing part of a configuration file or similar format,
* advance past comments and whitespace and try to parse a single line. If we
* parse a line successfully, set *<b>key_out</b> to a new string holding the
* key portion and *<b>value_out</b> to a new string holding the value portion
* of the line, and return a pointer to the start of the next line. If we run
* out of data, return a pointer to the end of the string. If we encounter an
* error, return NULL and set *<b>err_out</b> (if provided) to an error
* message.
*/
const char *
parse_config_line_from_str_verbose(const char *line, char **key_out,
char **value_out,
const char **err_out)
{
/*
See torrc_format.txt for a description of the (silly) format this parses.
*/
const char *key, *val, *cp;
int continuation = 0;
tor_assert(key_out);
tor_assert(value_out);
*key_out = *value_out = NULL;
key = val = NULL;
/* Skip until the first keyword. */
while (1) {
while (TOR_ISSPACE(*line))
++line;
if (*line == '#') {
while (*line && *line != '\n')
++line;
} else {
break;
}
}
if (!*line) { /* End of string? */
*key_out = *value_out = NULL;
return line;
}
/* Skip until the next space or \ followed by newline. */
key = line;
while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
! (line[0] == '\\' && line[1] == '\n'))
++line;
*key_out = tor_strndup(key, line-key);
/* Skip until the value. */
while (*line == ' ' || *line == '\t')
++line;
val = line;
/* Find the end of the line. */
if (*line == '\"') { // XXX No continuation handling is done here
if (!(line = unescape_string(line, value_out, NULL))) {
if (err_out)
*err_out = "Invalid escape sequence in quoted string";
return NULL;
}
while (*line == ' ' || *line == '\t')
++line;
if (*line == '\r' && *(++line) == '\n')
++line;
if (*line && *line != '#' && *line != '\n') {
if (err_out)
*err_out = "Excess data after quoted string";
return NULL;
}
} else {
/* Look for the end of the line. */
while (*line && *line != '\n' && (*line != '#' || continuation)) {
if (*line == '\\' && line[1] == '\n') {
continuation = 1;
line += 2;
} else if (*line == '#') {
do {
++line;
} while (*line && *line != '\n');
if (*line == '\n')
++line;
} else {
++line;
}
}
if (*line == '\n') {
cp = line++;
} else {
cp = line;
}
/* Now back cp up to be the last nonspace character */
while (cp>val && TOR_ISSPACE(*(cp-1)))
--cp;
tor_assert(cp >= val);
/* Now copy out and decode the value. */
*value_out = tor_strndup(val, cp-val);
if (continuation) {
char *v_out, *v_in;
v_out = v_in = *value_out;
while (*v_in) {
if (*v_in == '#') {
do {
++v_in;
} while (*v_in && *v_in != '\n');
if (*v_in == '\n')
++v_in;
} else if (v_in[0] == '\\' && v_in[1] == '\n') {
v_in += 2;
} else {
*v_out++ = *v_in++;
}
}
*v_out = '\0';
}
}
if (*line == '#') {
do {
++line;
} while (*line && *line != '\n');
}
while (TOR_ISSPACE(*line)) ++line;
return line;
}

47
src/common/confline.h Normal file
View file

@ -0,0 +1,47 @@
/* Copyright (c) 2001 Matej Pfajfar.
* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2017, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#ifndef TOR_CONFLINE_H
#define TOR_CONFLINE_H
/** Ordinary configuration line. */
#define CONFIG_LINE_NORMAL 0
/** Appends to previous configuration for the same option, even if we
* would ordinary replace it. */
#define CONFIG_LINE_APPEND 1
/* Removes all previous configuration for an option. */
#define CONFIG_LINE_CLEAR 2
/** A linked list of lines in a config file, or elsewhere */
typedef struct config_line_t {
char *key;
char *value;
struct config_line_t *next;
/** What special treatment (if any) does this line require? */
unsigned int command:2;
/** If true, subsequent assignments to this linelist should replace
* it, not extend it. Set only on the first item in a linelist in an
* or_options_t. */
unsigned int fragile:1;
} config_line_t;
void config_line_append(config_line_t **lst,
const char *key, const char *val);
config_line_t *config_lines_dup(const config_line_t *inp);
config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
const char *key);
const config_line_t *config_line_find(const config_line_t *lines,
const char *key);
int config_lines_eq(config_line_t *a, config_line_t *b);
int config_count_key(const config_line_t *a, const char *key);
int config_get_lines(const char *string, config_line_t **result, int extended);
void config_free_lines(config_line_t *front);
const char *parse_config_line_from_str_verbose(const char *line,
char **key_out, char **value_out,
const char **err_out);
#endif

2
src/common/include.am
View file

@ -84,6 +84,7 @@ LIBOR_A_SRC = \
src/common/compat.c \
src/common/compat_threads.c \
src/common/compat_time.c \
src/common/confline.c \
src/common/container.c \
src/common/log.c \
src/common/memarea.c \
@ -144,6 +145,7 @@ COMMONHEADERS = \
src/common/compat_openssl.h \
src/common/compat_threads.h \
src/common/compat_time.h \
src/common/confline.h \
src/common/container.h \
src/common/crypto.h \
src/common/crypto_curve25519.h \

160
src/common/storagedir.c
View file

@ -3,6 +3,8 @@
#include "container.h"
#include "compat.h"
#include "confline.h"
#include "memarea.h"
#include "sandbox.h"
#include "storagedir.h"
#include "torlog.h"
@ -237,6 +239,40 @@ find_unused_fname(storage_dir_t *d)
return NULL;
}
/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
* sized_chunk_t rather than a single byte array. */
static int
storage_dir_save_chunks_to_file(storage_dir_t *d,
const smartlist_t *chunks,
int binary,
char **fname_out)
{
uint64_t total_length = 0;
char *fname = find_unused_fname(d);
if (!fname)
return -1;
SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
total_length += ch->len);
char *path = NULL;
tor_asprintf(&path, "%s/%s", d->directory, fname);
int r = write_chunks_to_file(path, chunks, binary, 0);
if (r == 0) {
if (d->usage_known)
d->usage += total_length;
if (fname_out) {
*fname_out = tor_strdup(fname);
}
if (d->contents)
smartlist_add(d->contents, tor_strdup(fname));
}
tor_free(fname);
tor_free(path);
return r;
}
/** Try to write the <b>length</b> bytes at <b>data</b> into a new file
* in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a
* newly allocated string containing the filename. On failure, return
@ -248,25 +284,11 @@ storage_dir_save_bytes_to_file(storage_dir_t *d,
int binary,
char **fname_out)
{
char *fname = find_unused_fname(d);
if (!fname)
return -1;
char *path = NULL;
tor_asprintf(&path, "%s/%s", d->directory, fname);
int r = write_bytes_to_file(path, (const char *)data, length, binary);
if (r == 0) {
if (d->usage_known)
d->usage += length;
if (fname_out) {
*fname_out = tor_strdup(fname);
}
if (d->contents)
smartlist_add(d->contents, tor_strdup(fname));
}
tor_free(fname);
tor_free(path);
smartlist_t *chunks = smartlist_new();
sized_chunk_t chunk = { (const char *)data, length };
smartlist_add(chunks, &chunk);
int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
smartlist_free(chunks);
return r;
}
@ -284,6 +306,106 @@ storage_dir_save_string_to_file(storage_dir_t *d,
(const uint8_t*)str, strlen(str), binary, fname_out);
}
/**
* As storage_dir_save_bytes_to_file, but associates the data with the
* key-value pairs in <b>labels</b>. Files
* stored in this format can be recovered with storage_dir_map_labeled
* or storage_dir_read_labeled().
*/
int
storage_dir_save_labeled_to_file(storage_dir_t *d,
const config_line_t *labels,
const uint8_t *data,
size_t length,
char **fname_out)
{
/*
* The storage format is to prefix the data with the key-value pairs in
* <b>labels</b>, and a single NUL separator. But code outside this module
* MUST NOT rely on that format.
*/
smartlist_t *chunks = smartlist_new();
memarea_t *area = memarea_new();
const config_line_t *line;
for (line = labels; line; line = line->next) {
sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
const size_t allocated = sz->len + 1;
char *bytes = memarea_alloc(area, allocated);
tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
sz->bytes = bytes;
smartlist_add(chunks, sz);
}
sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
nul->len = 1;
nul->bytes = "\0";
smartlist_add(chunks, nul);
sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
datachunk->bytes = (const char *)data;
datachunk->len = length;
smartlist_add(chunks, datachunk);
int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
smartlist_free(chunks);
memarea_drop_all(area);
return r;
}
/**
* Map a file that was created with storage_dir_save_labeled(). On failure,
* return NULL. On success, write a set of newly allocated labels into to
* *<b>labels_out</b>, a pointer to the into *<b>data_out</b>, and the data's
* into *<b>sz_out</b>. On success, also return a tor_mmap_t object whose
* contents should not be used -- it needs to be kept around, though, for as
* long as <b>data_out</b> is going to be valid.
*/
tor_mmap_t *
storage_dir_map_labeled(storage_dir_t *dir,
const char *fname,
config_line_t **labels_out,
const uint8_t **data_out,
size_t *sz_out)
{
tor_mmap_t *m = storage_dir_map(dir, fname);
if (! m)
goto err;
const char *nulp = memchr(m->data, '\0', m->size);
if (! nulp)
goto err;
if (labels_out && config_get_lines(m->data, labels_out, 0) < 0)
goto err;
size_t offset = nulp - m->data + 1;
tor_assert(offset <= m->size);
*data_out = (const uint8_t *)(m->data + offset);
*sz_out = m->size - offset;
return m;
err:
tor_munmap_file(m);
return NULL;
}
/** As storage_dir_map_labeled, but return a new byte array containing the
* data. */
uint8_t *
storage_dir_read_labeled(storage_dir_t *dir,
const char *fname,
config_line_t **labels_out,
size_t *sz_out)
{
const uint8_t *data = NULL;
tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
&data, sz_out);
if (m == NULL)
return NULL;
uint8_t *result = tor_memdup(data, *sz_out);
tor_munmap_file(m);
return result;
}
/**
* Remove the file called <b>fname</b> from <b>d</b>.
*/

15
src/common/storagedir.h
View file

@ -5,7 +5,7 @@
#define TOR_STORAGEDIR_H
typedef struct storage_dir_t storage_dir_t;
struct config_line_t;
struct sandbox_cfg_elem;
storage_dir_t * storage_dir_new(const char *dirname, int n_files);
@ -26,6 +26,19 @@ int storage_dir_save_string_to_file(storage_dir_t *d,
const char *data,
int binary,
char **fname_out);
int storage_dir_save_labeled_to_file(storage_dir_t *d,
const struct config_line_t *labels,
const uint8_t *data,
size_t length,
char **fname_out);
tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir,
const char *fname,
struct config_line_t **labels_out,
const uint8_t **data_out,
size_t *size_out);
uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname,
struct config_line_t **labels_out,
size_t *sz_out);
void storage_dir_remove_file(storage_dir_t *d,
const char *fname);
int storage_dir_shrink(storage_dir_t *d,

131
src/common/util.c
View file

@ -3045,137 +3045,6 @@ unescape_string(const char *s, char **result, size_t *size_out)
}
}
/** Given a string containing part of a configuration file or similar format,
* advance past comments and whitespace and try to parse a single line. If we
* parse a line successfully, set *<b>key_out</b> to a new string holding the
* key portion and *<b>value_out</b> to a new string holding the value portion
* of the line, and return a pointer to the start of the next line. If we run
* out of data, return a pointer to the end of the string. If we encounter an
* error, return NULL and set *<b>err_out</b> (if provided) to an error
* message.
*/
const char *
parse_config_line_from_str_verbose(const char *line, char **key_out,
char **value_out,
const char **err_out)
{
/*
See torrc_format.txt for a description of the (silly) format this parses.
*/
const char *key, *val, *cp;
int continuation = 0;
tor_assert(key_out);
tor_assert(value_out);
*key_out = *value_out = NULL;
key = val = NULL;
/* Skip until the first keyword. */
while (1) {
while (TOR_ISSPACE(*line))
++line;
if (*line == '#') {
while (*line && *line != '\n')
++line;
} else {
break;
}
}
if (!*line) { /* End of string? */
*key_out = *value_out = NULL;
return line;
}
/* Skip until the next space or \ followed by newline. */
key = line;
while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
! (line[0] == '\\' && line[1] == '\n'))
++line;
*key_out = tor_strndup(key, line-key);
/* Skip until the value. */
while (*line == ' ' || *line == '\t')
++line;
val = line;
/* Find the end of the line. */
if (*line == '\"') { // XXX No continuation handling is done here
if (!(line = unescape_string(line, value_out, NULL))) {
if (err_out)
*err_out = "Invalid escape sequence in quoted string";
return NULL;
}
while (*line == ' ' || *line == '\t')
++line;
if (*line == '\r' && *(++line) == '\n')
++line;
if (*line && *line != '#' && *line != '\n') {
if (err_out)
*err_out = "Excess data after quoted string";
return NULL;
}
} else {
/* Look for the end of the line. */
while (*line && *line != '\n' && (*line != '#' || continuation)) {
if (*line == '\\' && line[1] == '\n') {
continuation = 1;
line += 2;
} else if (*line == '#') {
do {
++line;
} while (*line && *line != '\n');
if (*line == '\n')
++line;
} else {
++line;
}
}
if (*line == '\n') {
cp = line++;
} else {
cp = line;
}
/* Now back cp up to be the last nonspace character */
while (cp>val && TOR_ISSPACE(*(cp-1)))
--cp;
tor_assert(cp >= val);
/* Now copy out and decode the value. */
*value_out = tor_strndup(val, cp-val);
if (continuation) {
char *v_out, *v_in;
v_out = v_in = *value_out;
while (*v_in) {
if (*v_in == '#') {
do {
++v_in;
} while (*v_in && *v_in != '\n');
if (*v_in == '\n')
++v_in;
} else if (v_in[0] == '\\' && v_in[1] == '\n') {
v_in += 2;
} else {
*v_out++ = *v_in++;
}
}
*v_out = '\0';
}
}
if (*line == '#') {
do {
++line;
} while (*line && *line != '\n');
}
while (TOR_ISSPACE(*line)) ++line;
return line;
}
/** Expand any homedir prefix on <b>filename</b>; return a newly allocated
* string. */
char *

3
src/common/util.h
View file

@ -389,9 +389,6 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
size_t *sz_out)
ATTR_MALLOC;
const char *unescape_string(const char *s, char **result, size_t *size_out);
const char *parse_config_line_from_str_verbose(const char *line,
char **key_out, char **value_out,
const char **err_out);
char *expand_filename(const char *filename);
MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
int path_is_relative(const char *filename);

176
src/or/confparse.c
View file

@ -31,8 +31,6 @@ static int config_parse_msec_interval(const char *s, int *ok);
static int config_parse_interval(const char *s, int *ok);
static void config_reset(const config_format_t *fmt, void *options,
const config_var_t *var, int use_defaults);
static config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
const char *key);
/** Allocate an empty configuration object of a given format type. */
void *
@ -80,120 +78,6 @@ config_expand_abbrev(const config_format_t *fmt, const char *option,
return option;
}
/** Helper: allocate a new configuration option mapping 'key' to 'val',
* append it to *<b>lst</b>. */
void
config_line_append(config_line_t **lst,
const char *key,
const char *val)
{
config_line_t *newline;
newline = tor_malloc_zero(sizeof(config_line_t));
newline->key = tor_strdup(key);
newline->value = tor_strdup(val);
newline->next = NULL;
while (*lst)
lst = &((*lst)->next);
(*lst) = newline;
}
/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL
* if no such key exists. For handling commandline-only options only; other
* options should be looked up in the appropriate data structure. */
const config_line_t *
config_line_find(const config_line_t *lines,
const char *key)
{
const config_line_t *cl;
for (cl = lines; cl; cl = cl->next) {
if (!strcmp(cl->key, key))
return cl;
}
return NULL;
}
/** Helper: parse the config string and strdup into key/value
* strings. Set *result to the list, or NULL if parsing the string
* failed. Return 0 on success, -1 on failure. Warn and ignore any
* misformatted lines.
*
* If <b>extended</b> is set, then treat keys beginning with / and with + as
* indicating "clear" and "append" respectively. */
int
config_get_lines(const char *string, config_line_t **result, int extended)
{
config_line_t *list = NULL, **next;
char *k, *v;
const char *parse_err;
next = &list;
do {
k = v = NULL;
string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
if (!string) {
log_warn(LD_CONFIG, "Error while parsing configuration: %s",
parse_err?parse_err:"<unknown>");
config_free_lines(list);
tor_free(k);
tor_free(v);
return -1;
}
if (k && v) {
unsigned command = CONFIG_LINE_NORMAL;
if (extended) {
if (k[0] == '+') {
char *k_new = tor_strdup(k+1);
tor_free(k);
k = k_new;
command = CONFIG_LINE_APPEND;
} else if (k[0] == '/') {
char *k_new = tor_strdup(k+1);
tor_free(k);
k = k_new;
tor_free(v);
v = tor_strdup("");
command = CONFIG_LINE_CLEAR;
}
}
/* This list can get long, so we keep a pointer to the end of it
* rather than using config_line_append over and over and getting
* n^2 performance. */
*next = tor_malloc_zero(sizeof(config_line_t));
(*next)->key = k;
(*next)->value = v;
(*next)->next = NULL;
(*next)->command = command;
next = &((*next)->next);
} else {
tor_free(k);
tor_free(v);
}
} while (*string);
*result = list;
return 0;
}
/**
* Free all the configuration lines on the linked list <b>front</b>.
*/
void
config_free_lines(config_line_t *front)
{
config_line_t *tmp;
while (front) {
tmp = front;
front = tmp->next;
tor_free(tmp->key);
tor_free(tmp->value);
tor_free(tmp);
}
}
/** If <b>key</b> is a deprecated configuration option, return the message
* explaining why it is deprecated (which may be an empty string). Return NULL
* if it is not deprecated. The <b>key</b> field must be fully expanded. */
@ -633,36 +517,6 @@ config_value_needs_escape(const char *value)
return 0;
}
/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
config_line_t *
config_lines_dup(const config_line_t *inp)
{
return config_lines_dup_and_filter(inp, NULL);
}
/** Return a newly allocated deep copy of the lines in <b>inp</b>,
* but only the ones that match <b>key</b>. */
static config_line_t *
config_lines_dup_and_filter(const config_line_t *inp,
const char *key)
{
config_line_t *result = NULL;
config_line_t **next_out = &result;
while (inp) {
if (key && strcasecmpstart(inp->key, key)) {
inp = inp->next;
continue;
}
*next_out = tor_malloc_zero(sizeof(config_line_t));
(*next_out)->key = tor_strdup(inp->key);
(*next_out)->value = tor_strdup(inp->value);
inp = inp->next;
next_out = &((*next_out)->next);
}
(*next_out) = NULL;
return result;
}
/** Return newly allocated line or lines corresponding to <b>key</b> in the
* configuration <b>options</b>. If <b>escape_val</b> is true and a
* value needs to be quoted before it's put in a config file, quote and
@ -1028,36 +882,6 @@ config_free(const config_format_t *fmt, void *options)
tor_free(options);
}
/** Return true iff a and b contain identical keys and values in identical
* order. */
int
config_lines_eq(config_line_t *a, config_line_t *b)
{
while (a && b) {
if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
return 0;
a = a->next;
b = b->next;
}
if (a || b)
return 0;
return 1;
}
/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
int
config_count_key(const config_line_t *a, const char *key)
{
int n = 0;
while (a) {
if (!strcasecmp(a->key, key)) {
++n;
}
a = a->next;
}
return n;
}
/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
* and <b>o2</b>. Must not be called for LINELIST_S or OBSOLETE options.
*/

9
src/or/confparse.h
View file

@ -103,14 +103,7 @@ typedef struct config_format_t {
#define CAL_WARN_DEPRECATIONS (1u<<2)
void *config_new(const config_format_t *fmt);
void config_line_append(config_line_t **lst,
const char *key, const char *val);
config_line_t *config_lines_dup(const config_line_t *inp);
const config_line_t *config_line_find(const config_line_t *lines,
const char *key);
void config_free(const config_format_t *fmt, void *options);
int config_lines_eq(config_line_t *a, config_line_t *b);
int config_count_key(const config_line_t *a, const char *key);
config_line_t *config_get_assigned_option(const config_format_t *fmt,
const void *options, const char *key,
int escape_val);
@ -132,8 +125,6 @@ const char *config_find_deprecation(const config_format_t *fmt,
const config_var_t *config_find_option(const config_format_t *fmt,
const char *key);
int config_get_lines(const char *string, config_line_t **result, int extended);
void config_free_lines(config_line_t *front);
const char *config_expand_abbrev(const config_format_t *fmt,
const char *option,
int command_line, int warn_obsolete);

506
src/or/conscache.c Normal file
View file

@ -0,0 +1,506 @@
/* Copyright (c) 2017, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#include "or.h"
#include "config.h"
#include "conscache.h"
#include "storagedir.h"
#define CCE_MAGIC 0x17162253
/**
* A consensus_cache_entry_t is a reference-counted handle to an
* item in a consensus_cache_t. It can be mmapped into RAM, or not,
* depending whether it's currently in use.
*/
struct consensus_cache_entry_t {
uint32_t magic; /**< Must be set to CCE_MAGIC */
int32_t refcnt; /**< Reference count. */
unsigned can_remove : 1; /**< If true, we want to delete this file. */
/** If true, we intend to unmap this file as soon as we're done with it. */
unsigned release_aggressively : 1;
/** Filename for this object within the storage_dir_t */
char *fname;
/** Labels associated with this object. Immutable once the object
* is created. */
config_line_t *labels;
/** Pointer to the cache that includes this entry (if any). */
consensus_cache_t *in_cache;
/** Since what time has this object been mapped into RAM, but with the cache
* being the only having a reference to it? */
time_t unused_since;
/** mmaped contents of the underlying file. May be NULL */
tor_mmap_t *map;
/** Length of the body within <b>map</b>. */
size_t bodylen;
/** Pointer to the body within <b>map</b>. */
const uint8_t *body;
};
/**
* A consensus_cache_t holds a directory full of labeled items.
*/
struct consensus_cache_t {
/** Underling storage_dir_t to handle persistence */
storage_dir_t *dir;
/** List of all the entries in the directory. */
smartlist_t *entries;
};
static void consensus_cache_clear(consensus_cache_t *cache);
static void consensus_cache_rescan(consensus_cache_t *);
static void consensus_cache_entry_map(consensus_cache_t *,
consensus_cache_entry_t *);
static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
/**
* Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
* data directory, to hold up to <b>max_entries</b> of data.
*/
consensus_cache_t *
consensus_cache_open(const char *subdir, int max_entries)
{
consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
char *directory = get_datadir_fname(subdir);
cache->dir = storage_dir_new(directory, max_entries);
tor_free(directory);
if (!cache->dir) {
tor_free(cache);
return NULL;
}
consensus_cache_rescan(cache);
return cache;
}
/**
* Helper: clear all entries from <b>cache</b> (but do not delete
* any that aren't marked for removal
*/
static void
consensus_cache_clear(consensus_cache_t *cache)
{
consensus_cache_delete_pending(cache);
SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
ent->in_cache = NULL;
consensus_cache_entry_decref(ent);
} SMARTLIST_FOREACH_END(ent);
smartlist_free(cache->entries);
cache->entries = NULL;
}
/**
* Drop all storage held by <b>cache</b>.
*/
void
consensus_cache_free(consensus_cache_t *cache)
{
if (! cache)
return;
if (cache->entries) {
consensus_cache_clear(cache);
}
storage_dir_free(cache->dir);
tor_free(cache);
}
/**
* Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
* labeling that data with <b>labels</b>. On failure, return NULL. On
* success, return a newly created consensus_cache_entry_t.
*
* The returned value will be owned by the cache, and you will have a
* reference to it. Call consensus_cache_entry_decref() when you are
* done with it.
*
* The provided <b>labels</b> MUST have distinct keys: if they don't,
* this API does not specify which values (if any) for the duplicate keys
* will be considered.
*/
consensus_cache_entry_t *
consensus_cache_add(consensus_cache_t *cache,
const config_line_t *labels,
const uint8_t *data,
size_t datalen)
{
char *fname = NULL;
int r = storage_dir_save_labeled_to_file(cache->dir,
labels, data, datalen, &fname);
if (r < 0 || fname == NULL) {
return NULL;
}
consensus_cache_entry_t *ent =
tor_malloc_zero(sizeof(consensus_cache_entry_t));
ent->magic = CCE_MAGIC;
ent->fname = fname;
ent->labels = config_lines_dup(labels);
ent->in_cache = cache;
ent->unused_since = TIME_MAX;
smartlist_add(cache->entries, ent);
/* Start the reference count at 2: the caller owns one copy, and the
* cache owns another.
*/
ent->refcnt = 2;
return ent;
}
/**
* Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
* Return NULL if no such entry exists.
*
* Does not adjust reference counts.
*/
consensus_cache_entry_t *
consensus_cache_find_first(consensus_cache_t *cache,
const char *key,
const char *value)
{
smartlist_t *tmp = smartlist_new();
consensus_cache_find_all(tmp, cache, key, value);
consensus_cache_entry_t *ent = NULL;
if (smartlist_len(tmp))
ent = smartlist_get(tmp, 0);
smartlist_free(tmp);
return ent;
}
/**
* Given a <b>cache</b>, add every entry to <b>out<b> for which
* <b>key</b>=<b>value</b>. If <b>key</b> is NULL, add every entry.
*
* Does not adjust reference counts.
*/
void
consensus_cache_find_all(smartlist_t *out,
consensus_cache_t *cache,
const char *key,
const char *value)
{
if (! key) {
smartlist_add_all(out, cache->entries);
return;
}
SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
const char *found_val = consensus_cache_entry_get_value(ent, key);
if (found_val && !strcmp(value, found_val)) {
smartlist_add(out, ent);
}
} SMARTLIST_FOREACH_END(ent);
}
/**
* Given a list of consensus_cache_entry_t, remove all those entries
* that do not have <b>key</b>=<b>value</b> in their labels.
*
* Does not adjust reference counts.
*/
void
consensus_cache_filter_list(smartlist_t *lst,
const char *key,
const char *value)
{
if (BUG(lst == NULL))
return; // LCOV_EXCL_LINE
if (key == NULL)
return;
SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
const char *found_val = consensus_cache_entry_get_value(ent, key);
if (! found_val || strcmp(value, found_val)) {
SMARTLIST_DEL_CURRENT(lst, ent);
}
} SMARTLIST_FOREACH_END(ent);
}
/**
* If <b>ent</b> has a label with the given <b>key</b>, return its
* value. Otherwise return NULL.
*
* The return value is only guaranteed to be valid for as long as you
* hold a reference to <b>ent</b>.
*/
const char *
consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
const char *key)
{
const config_line_t *match = config_line_find(ent->labels, key);
if (match)
return match->value;
else
return NULL;
}
/**
* Return a pointer to the labels in <b>ent</b>.
*
* This pointer is only guaranteed to be valid for as long as you
* hold a reference to <b>ent</b>.
*/
const config_line_t *
consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
{
return ent->labels;
}
/**
* Increase the reference count of <b>ent</b>.
*/
void
consensus_cache_entry_incref(consensus_cache_entry_t *ent)
{
if (BUG(ent->magic != CCE_MAGIC))
return; // LCOV_EXCL_LINE
++ent->refcnt;
ent->unused_since = TIME_MAX;
}
/**
* Release a reference held to <b>ent</b>.
*
* If it was the last reference, ent will be freed. Therefore, you must not
* use <b>ent</b> after calling this function.
*/
void
consensus_cache_entry_decref(consensus_cache_entry_t *ent)
{
if (! ent)
return;
if (BUG(ent->refcnt <= 0))
return; // LCOV_EXCL_LINE
if (BUG(ent->magic != CCE_MAGIC))
return; // LCOV_EXCL_LINE
--ent->refcnt;
if (ent->refcnt == 1 && ent->in_cache) {
/* Only the cache has a reference: we don't need to keep the file
* mapped */
if (ent->map) {
if (ent->release_aggressively) {
consensus_cache_entry_unmap(ent);
} else {
ent->unused_since = approx_time();
}
}
return;
}
if (ent->refcnt > 0)
return;
/* Refcount is zero; we can free it. */
if (ent->map) {
consensus_cache_entry_unmap(ent);
}
tor_free(ent->fname);
config_free_lines(ent->labels);
memwipe(ent, 0, sizeof(consensus_cache_entry_t));
tor_free(ent);
}
/**
* Mark <b>ent</b> for deletion from the cache. Deletion will not occur
* until the cache is the only place that holds a reference to <b>ent</b>.
*/
void
consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
{
ent->can_remove = 1;
}
/**
* Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
* any longer than we're actually using it.
*/
void
consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
{
ent->release_aggressively = 1;
}
/**
* Try to read the body of <b>ent</b> into memory if it isn't already
* loaded. On success, set *<b>body_out</b> to the body, *<b>sz_out</b>
* to its size, and return 0. On failure return -1.
*
* The resulting body pointer will only be valid for as long as you
* hold a reference to <b>ent</b>.
*/
int
consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
const uint8_t **body_out,
size_t *sz_out)
{
if (BUG(ent->magic != CCE_MAGIC))
return -1; // LCOV_EXCL_LINE
if (! ent->map) {
if (! ent->in_cache)
return -1;
consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
(consensus_cache_entry_t *)ent);
if (! ent->map) {
return -1;
}
}
*body_out = ent->body;
*sz_out = ent->bodylen;
return 0;
}
/**
* Unmap every mmap'd element of <b>cache</b> that has been unused
* since <b>cutoff</b>.
*/
void
consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
{
SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
tor_assert_nonfatal(ent->in_cache == cache);
if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
/* Somebody is using this entry right now */
continue;
}
if (ent->unused_since > cutoff) {
/* Has been unused only for a little while */
continue;
}
if (ent->map == NULL) {
/* Not actually mapped. */
continue;
}
consensus_cache_entry_unmap(ent);
} SMARTLIST_FOREACH_END(ent);
}
/**
* Delete every element of <b>cache</b> has been marked with
* consensus_cache_entry_mark_for_removal, and which is not in use except by
* the cache.
*/
void
consensus_cache_delete_pending(consensus_cache_t *cache)
{
SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
tor_assert_nonfatal(ent->in_cache == cache);
if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
/* Somebody is using this entry right now */
continue;
}
if (ent->can_remove == 0) {
/* Don't want to delete this. */
continue;
}
if (BUG(ent->refcnt <= 0)) {
continue; // LCOV_EXCL_LINE
}
SMARTLIST_DEL_CURRENT(cache->entries, ent);
ent->in_cache = NULL;
char *fname = tor_strdup(ent->fname); /* save a copy */
consensus_cache_entry_decref(ent);
storage_dir_remove_file(cache->dir, fname);
tor_free(fname);
} SMARTLIST_FOREACH_END(ent);
}
/**
* Internal helper: rescan <b>cache</b> and rebuild its list of entries.
*/
static void
consensus_cache_rescan(consensus_cache_t *cache)
{
if (cache->entries) {
consensus_cache_clear(cache);
}
cache->entries = smartlist_new();
const smartlist_t *fnames = storage_dir_list(cache->dir);
SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
tor_mmap_t *map = NULL;
config_line_t *labels = NULL;
const uint8_t *body;
size_t bodylen;
map = storage_dir_map_labeled(cache->dir, fname,
&labels, &body, &bodylen);
if (! map) {
/* Can't load this; continue */
log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
escaped(fname), strerror(errno));
continue;
}
consensus_cache_entry_t *ent =
tor_malloc_zero(sizeof(consensus_cache_entry_t));
ent->magic = CCE_MAGIC;
ent->fname = tor_strdup(fname);
ent->labels = labels;
ent->refcnt = 1;
ent->in_cache = cache;
ent->unused_since = TIME_MAX;
smartlist_add(cache->entries, ent);
tor_munmap_file(map); /* don't actually need to keep this around */
} SMARTLIST_FOREACH_END(fname);
}
/**
* Make sure that <b>ent</b> is mapped into RAM.
*/
static void
consensus_cache_entry_map(consensus_cache_t *cache,
consensus_cache_entry_t *ent)
{
if (ent->map)
return;
ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
NULL, &ent->body, &ent->bodylen);
ent->unused_since = TIME_MAX;
}
/**
* Unmap <b>ent</b> from RAM.
*
* Do not call this if something other than the cache is holding a reference
* to <b>ent</b>
*/
static void
consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
{
ent->unused_since = TIME_MAX;
if (!ent->map)
return;
tor_munmap_file(ent->map);
ent->map = NULL;
ent->body = NULL;
ent->bodylen = 0;
ent->unused_since = TIME_MAX;
}
#ifdef TOR_UNIT_TESTS
/**
* Testing only: Return true iff <b>ent</b> is mapped into memory.
*
* (In normal operation, this information is not exposed.)
*/
int
consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
{
if (ent->map) {
tor_assert(ent->body);
return 1;
} else {
tor_assert(!ent->body);
return 0;
}
}
#endif

52
src/or/conscache.h Normal file
View file

@ -0,0 +1,52 @@
/* Copyright (c) 2017, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#ifndef TOR_CONSCACHE_H
#define TOR_CONSCACHE_H
typedef struct consensus_cache_entry_t consensus_cache_entry_t;
typedef struct consensus_cache_t consensus_cache_t;
consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
void consensus_cache_free(consensus_cache_t *cache);
void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
void consensus_cache_delete_pending(consensus_cache_t *cache);
consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
const config_line_t *labels,
const uint8_t *data,
size_t datalen);
consensus_cache_entry_t *consensus_cache_find_first(
consensus_cache_t *cache,
const char *key,
const char *value);
void consensus_cache_find_all(smartlist_t *out,
consensus_cache_t *cache,
const char *key,
const char *value);
void consensus_cache_filter_list(smartlist_t *lst,
const char *key,
const char *value);
const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
const char *key);
const config_line_t *consensus_cache_entry_get_labels(
const consensus_cache_entry_t *ent);
void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
void consensus_cache_entry_decref(consensus_cache_entry_t *ent);
void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
void consensus_cache_entry_mark_for_aggressive_release(
consensus_cache_entry_t *ent);
int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
const uint8_t **body_out,
size_t *sz_out);
#ifdef TOR_UNIT_TESTS
int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);
#endif
#endif

2
src/or/include.am
View file

@ -36,6 +36,7 @@ LIBTOR_A_SOURCES = \
src/or/connection.c \
src/or/connection_edge.c \
src/or/connection_or.c \
src/or/conscache.c \
src/or/consdiff.c \
src/or/control.c \
src/or/cpuworker.c \
@ -152,6 +153,7 @@ ORHEADERS = \
src/or/connection.h \
src/or/connection_edge.h \
src/or/connection_or.h \
src/or/conscache.h \
src/or/consdiff.h \
src/or/control.h \
src/or/cpuworker.h \

22
src/or/or.h
View file

@ -75,6 +75,7 @@
#include "address.h"
#include "compat_libevent.h"
#include "ht.h"
#include "confline.h"
#include "replaycache.h"
#include "crypto_curve25519.h"
#include "crypto_ed25519.h"
@ -3529,27 +3530,6 @@ typedef struct port_cfg_t {
char unix_addr[FLEXIBLE_ARRAY_MEMBER];
} port_cfg_t;
/** Ordinary configuration line. */
#define CONFIG_LINE_NORMAL 0
/** Appends to previous configuration for the same option, even if we
* would ordinary replace it. */
#define CONFIG_LINE_APPEND 1
/* Removes all previous configuration for an option. */
#define CONFIG_LINE_CLEAR 2
/** A linked list of lines in a config file. */
typedef struct config_line_t {
char *key;
char *value;
struct config_line_t *next;
/** What special treatment (if any) does this line require? */
unsigned int command:2;
/** If true, subsequent assignments to this linelist should replace
* it, not extend it. Set only on the first item in a linelist in an
* or_options_t. */
unsigned int fragile:1;
} config_line_t;
typedef struct routerset_t routerset_t;
/** A magic value for the (Socks|OR|...)Port options below, telling Tor

1
src/test/include.am
View file

@ -87,6 +87,7 @@ src_test_test_SOURCES = \
src/test/test_compat_libevent.c \
src/test/test_config.c \
src/test/test_connection.c \
src/test/test_conscache.c \
src/test/test_consdiff.c \
src/test/test_containers.c \
src/test/test_controller.c \

1
src/test/test.c
View file

@ -1195,6 +1195,7 @@ struct testgroup_t testgroups[] = {
{ "compat/libevent/", compat_libevent_tests },
{ "config/", config_tests },
{ "connection/", connection_tests },
{ "conscache/", conscache_tests },
{ "consdiff/", consdiff_tests },
{ "container/", container_tests },
{ "control/", controller_tests },

1
src/test/test.h
View file

@ -190,6 +190,7 @@ extern struct testcase_t circuituse_tests[];
extern struct testcase_t compat_libevent_tests[];
extern struct testcase_t config_tests[];
extern struct testcase_t connection_tests[];
extern struct testcase_t conscache_tests[];
extern struct testcase_t consdiff_tests[];
extern struct testcase_t container_tests[];
extern struct testcase_t controller_tests[];

341
src/test/test_conscache.c Normal file
View file

@ -0,0 +1,341 @@
/* Copyright (c) 2017, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#include "or.h"
#include "config.h"
#include "conscache.h"
#include "test.h"
#ifdef HAVE_UTIME_H
#include <utime.h>
#endif
static void
test_conscache_open_failure(void *arg)
{
(void) arg;
/* Try opening a directory that doesn't exist and which we shouldn't be
* able to create. */
consensus_cache_t *cache = consensus_cache_open("a/b/c/d/e/f/g", 128);
tt_ptr_op(cache, OP_EQ, NULL);
done:
;
}
static void
test_conscache_simple_usage(void *arg)
{
(void)arg;
consensus_cache_entry_t *ent = NULL, *ent2 = NULL;
/* Make a temporary datadir for these tests */
char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
tor_free(get_options_mutable()->DataDirectory);
get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
check_private_dir(ddir_fname, CPD_CREATE, NULL);
consensus_cache_t *cache = consensus_cache_open("cons", 128);
tt_assert(cache);
/* Create object; make sure it exists. */
config_line_t *labels = NULL;
config_line_append(&labels, "Hello", "world");
config_line_append(&labels, "Adios", "planetas");
ent = consensus_cache_add(cache,
labels, (const uint8_t *)"A\0B\0C", 5);
config_free_lines(labels);
labels = NULL;
tt_assert(ent);
/* Make a second object */
config_line_append(&labels, "Hello", "mundo");
config_line_append(&labels, "Adios", "planets");
ent2 = consensus_cache_add(cache,
labels, (const uint8_t *)"xyzzy", 5);
config_free_lines(labels);
labels = NULL;
tt_assert(ent2);
tt_assert(! consensus_cache_entry_is_mapped(ent2));
consensus_cache_entry_decref(ent2);
ent2 = NULL;
/* Check get_value */
tt_ptr_op(NULL, OP_EQ, consensus_cache_entry_get_value(ent, "hebbo"));
tt_str_op("world", OP_EQ, consensus_cache_entry_get_value(ent, "Hello"));
/* Check find_first */
ent2 = consensus_cache_find_first(cache, "Hello", "world!");
tt_ptr_op(ent2, OP_EQ, NULL);
ent2 = consensus_cache_find_first(cache, "Hello", "world");
tt_ptr_op(ent2, OP_EQ, ent);
ent2 = consensus_cache_find_first(cache, "Hello", "mundo");
tt_ptr_op(ent2, OP_NE, ent);
tt_assert(! consensus_cache_entry_is_mapped(ent));
/* Check get_body */
const uint8_t *bp = NULL;
size_t sz = 0;
int r = consensus_cache_entry_get_body(ent, &bp, &sz);
tt_int_op(r, OP_EQ, 0);
tt_u64_op(sz, OP_EQ, 5);
tt_mem_op(bp, OP_EQ, "A\0B\0C", 5);
tt_assert(consensus_cache_entry_is_mapped(ent));
/* Free and re-create the cache, to rescan the directory. */
consensus_cache_free(cache);
consensus_cache_entry_decref(ent);
cache = consensus_cache_open("cons", 128);
/* Make sure the entry is still there */
ent = consensus_cache_find_first(cache, "Hello", "mundo");
tt_assert(ent);
ent2 = consensus_cache_find_first(cache, "Adios", "planets");
tt_ptr_op(ent, OP_EQ, ent2);
consensus_cache_entry_incref(ent);
tt_assert(! consensus_cache_entry_is_mapped(ent));
r = consensus_cache_entry_get_body(ent, &bp, &sz);
tt_int_op(r, OP_EQ, 0);
tt_u64_op(sz, OP_EQ, 5);
tt_mem_op(bp, OP_EQ, "xyzzy", 5);
tt_assert(consensus_cache_entry_is_mapped(ent));
/* There should be two entries total. */
smartlist_t *entries = smartlist_new();
consensus_cache_find_all(entries, cache, NULL, NULL);
int n = smartlist_len(entries);
smartlist_free(entries);
tt_int_op(n, OP_EQ, 2);
done:
consensus_cache_entry_decref(ent);
tor_free(ddir_fname);
consensus_cache_free(cache);
}
static void
test_conscache_cleanup(void *arg)
{
(void)arg;
const int N = 20;
consensus_cache_entry_t **ents =
tor_calloc(N, sizeof(consensus_cache_entry_t*));
/* Make a temporary datadir for these tests */
char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
tor_free(get_options_mutable()->DataDirectory);
get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
check_private_dir(ddir_fname, CPD_CREATE, NULL);
consensus_cache_t *cache = consensus_cache_open("cons", 128);
tt_assert(cache);
/* Create a bunch of entries. */
int i;
for (i = 0; i < N; ++i) {
config_line_t *labels = NULL;
char num[8];
tor_snprintf(num, sizeof(num), "%d", i);
config_line_append(&labels, "test-id", "cleanup");
config_line_append(&labels, "index", num);
size_t bodylen = i * 3;
uint8_t *body = tor_malloc(bodylen);
memset(body, i, bodylen);
ents[i] = consensus_cache_add(cache, labels, body, bodylen);
tor_free(body);
config_free_lines(labels);
tt_assert(ents[i]);
/* We're still holding a reference to each entry at this point. */
}
/* Page all of the entries into RAM */
for (i = 0; i < N; ++i) {
const uint8_t *bp;
size_t sz;
tt_assert(! consensus_cache_entry_is_mapped(ents[i]));
consensus_cache_entry_get_body(ents[i], &bp, &sz);
tt_assert(consensus_cache_entry_is_mapped(ents[i]));
}
/* Mark some of the entries as deletable. */
for (i = 7; i < N; i += 7) {
consensus_cache_entry_mark_for_removal(ents[i]);
tt_assert(consensus_cache_entry_is_mapped(ents[i]));
}
/* Mark some of the entries as aggressively unpaged. */
for (i = 3; i < N; i += 3) {
consensus_cache_entry_mark_for_aggressive_release(ents[i]);
tt_assert(consensus_cache_entry_is_mapped(ents[i]));
}
/* Incref some of the entries again */
for (i = 0; i < N; i += 2) {
consensus_cache_entry_incref(ents[i]);
}
/* Now we're going to decref everything. We do so at a specific time. I'm
* picking the moment when I was writing this test, at 2017-04-05 12:16:48
* UTC. */
const time_t example_time = 1491394608;
update_approx_time(example_time);
for (i = 0; i < N; ++i) {
consensus_cache_entry_decref(ents[i]);
if (i % 2) {
ents[i] = NULL; /* We're no longer holding any reference here. */
}
}
/* At this point, the aggressively-released items with refcount 1 should
* be unmapped. Nothing should be deleted. */
consensus_cache_entry_t *e_tmp;
e_tmp = consensus_cache_find_first(cache, "index", "3");
tt_assert(e_tmp);
tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
e_tmp = consensus_cache_find_first(cache, "index", "5");
tt_assert(e_tmp);
tt_assert(consensus_cache_entry_is_mapped(e_tmp));
e_tmp = consensus_cache_find_first(cache, "index", "6");
tt_assert(e_tmp);
tt_assert(consensus_cache_entry_is_mapped(e_tmp));
e_tmp = consensus_cache_find_first(cache, "index", "7");
tt_assert(e_tmp);
tt_assert(consensus_cache_entry_is_mapped(e_tmp));
/* Delete the pending-deletion items. */
consensus_cache_delete_pending(cache);
{
smartlist_t *entries = smartlist_new();
consensus_cache_find_all(entries, cache, NULL, NULL);
int n = smartlist_len(entries);
smartlist_free(entries);
tt_int_op(n, OP_EQ, 20 - 1); /* 1 entry was deleted */
}
e_tmp = consensus_cache_find_first(cache, "index", "7"); // refcnt == 1...
tt_assert(e_tmp == NULL); // so deleted.
e_tmp = consensus_cache_find_first(cache, "index", "14"); // refcnt == 2
tt_assert(e_tmp); // so, not deleted.
/* Now do lazy unmapping. */
// should do nothing.
consensus_cache_unmap_lazy(cache, example_time - 10);
e_tmp = consensus_cache_find_first(cache, "index", "11");
tt_assert(e_tmp);
tt_assert(consensus_cache_entry_is_mapped(e_tmp));
// should actually unmap
consensus_cache_unmap_lazy(cache, example_time + 10);
e_tmp = consensus_cache_find_first(cache, "index", "11");
tt_assert(e_tmp);
tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
// This one will still be mapped, since it has a reference.
e_tmp = consensus_cache_find_first(cache, "index", "16");
tt_assert(e_tmp);
tt_assert(consensus_cache_entry_is_mapped(e_tmp));
for (i = 0; i < N; ++i) {
consensus_cache_entry_decref(ents[i]);
ents[i] = NULL;
}
/* Free and re-create the cache, to rescan the directory. Make sure the
* deleted thing is still deleted, along with the other deleted thing. */
consensus_cache_free(cache);
cache = consensus_cache_open("cons", 128);
{
smartlist_t *entries = smartlist_new();
consensus_cache_find_all(entries, cache, NULL, NULL);
int n = smartlist_len(entries);
smartlist_free(entries);
tt_int_op(n, OP_EQ, 18);
}
done:
for (i = 0; i < N; ++i) {
consensus_cache_entry_decref(ents[i]);
}
tor_free(ents);
tor_free(ddir_fname);
consensus_cache_free(cache);
}
static void
test_conscache_filter(void *arg)
{
(void)arg;
const int N = 30;
smartlist_t *lst = NULL;
/* Make a temporary datadir for these tests */
char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
tor_free(get_options_mutable()->DataDirectory);
get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
check_private_dir(ddir_fname, CPD_CREATE, NULL);
consensus_cache_t *cache = consensus_cache_open("cons", 128);
tt_assert(cache);
/* Create a bunch of entries with different labels */
int i;
for (i = 0; i < N; ++i) {
config_line_t *labels = NULL;
char num[8];
tor_snprintf(num, sizeof(num), "%d", i);
config_line_append(&labels, "test-id", "filter");
config_line_append(&labels, "index", num);
tor_snprintf(num, sizeof(num), "%d", i % 3);
config_line_append(&labels, "mod3", num);
tor_snprintf(num, sizeof(num), "%d", i % 5);
config_line_append(&labels, "mod5", num);
size_t bodylen = i * 3;
uint8_t *body = tor_malloc(bodylen);
memset(body, i, bodylen);
consensus_cache_entry_t *ent =
consensus_cache_add(cache, labels, body, bodylen);
tor_free(body);
config_free_lines(labels);
tt_assert(ent);
consensus_cache_entry_decref(ent);
}
lst = smartlist_new();
/* Find nothing. */
consensus_cache_find_all(lst, cache, "mod5", "5");
tt_int_op(smartlist_len(lst), OP_EQ, 0);
/* Find everything. */
consensus_cache_find_all(lst, cache, "test-id", "filter");
tt_int_op(smartlist_len(lst), OP_EQ, N);
/* Now filter to find the entries that have i%3 == 1 */
consensus_cache_filter_list(lst, "mod3", "1");
tt_int_op(smartlist_len(lst), OP_EQ, 10);
/* Now filter to find the entries that also have i%5 == 3 */
consensus_cache_filter_list(lst, "mod5", "3");
tt_int_op(smartlist_len(lst), OP_EQ, 2);
/* So now we have those entries for which i%15 == 13. */
consensus_cache_entry_t *ent1 = smartlist_get(lst, 0);
consensus_cache_entry_t *ent2 = smartlist_get(lst, 1);
const char *idx1 = consensus_cache_entry_get_value(ent1, "index");
const char *idx2 = consensus_cache_entry_get_value(ent2, "index");
tt_assert( (!strcmp(idx1, "28") && !strcmp(idx2, "13")) ||
(!strcmp(idx1, "13") && !strcmp(idx2, "28")) );
done:
tor_free(ddir_fname);
consensus_cache_free(cache);
smartlist_free(lst);
}
#define ENT(name) \
{ #name, test_conscache_ ## name, TT_FORK, NULL, NULL }
struct testcase_t conscache_tests[] = {
ENT(open_failure),
ENT(simple_usage),
ENT(cleanup),
ENT(filter),
END_OF_TESTCASES
};

105
src/test/test_storagedir.c
View file

@ -256,6 +256,109 @@ test_storagedir_cleaning(void *arg)
}
}
static void
test_storagedir_save_labeled(void *arg)
{
(void)arg;
char *dirname = tor_strdup(get_fname_rnd("store_dir"));
storage_dir_t *d = NULL;
uint8_t *inp = tor_malloc_zero(8192);
config_line_t *labels = NULL;
char *fname = NULL;
uint8_t *saved = NULL;
d = storage_dir_new(dirname, 10);
tt_assert(d);
crypto_rand((char *)inp, 8192);
config_line_append(&labels, "Foo", "bar baz");
config_line_append(&labels, "quux", "quuzXxz");
const char expected[] =
"Foo bar baz\n"
"quux quuzXxz\n";
int r = storage_dir_save_labeled_to_file(d, labels, inp, 8192, &fname);
tt_int_op(r, OP_EQ, 0);
size_t n;
saved = storage_dir_read(d, fname, 1, &n);
tt_assert(memchr(saved, '\0', n));
tt_str_op((char*)saved, OP_EQ, expected); /* NUL guarantees strcmp works */
tt_mem_op(saved+strlen(expected)+1, OP_EQ, inp, 8192);
done:
storage_dir_free(d);
tor_free(dirname);
tor_free(inp);
tor_free(fname);
config_free_lines(labels);
tor_free(saved);
}
static void
test_storagedir_read_labeled(void *arg)
{
(void)arg;
char *dirname = tor_strdup(get_fname_rnd("store_dir"));
storage_dir_t *d = NULL;
uint8_t *inp = tor_malloc_zero(8192);
config_line_t *labels = NULL, *labels2 = NULL;
char *fname = NULL;
tor_mmap_t *map = NULL;
uint8_t *as_read = NULL;
d = storage_dir_new(dirname, 10);
tt_assert(d);
tor_snprintf((char*)inp, 8192,
"Hello world\n"
"This is a test\n"
"Yadda yadda.\n");
size_t bodylen = 8192 - strlen((char*)inp) - 1;
crypto_rand((char *)inp+strlen((char*)inp)+1, bodylen);
int r = storage_dir_save_bytes_to_file(d, inp, 8192, 1, &fname);
tt_int_op(r, OP_EQ, 0);
/* Try mapping */
const uint8_t *datap = NULL;
size_t sz = 0;
map = storage_dir_map_labeled(d, fname, &labels, &datap, &sz);
tt_assert(map);
tt_assert(datap);
tt_u64_op(sz, OP_EQ, bodylen);
tt_mem_op(datap, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
tt_assert(labels);
tt_str_op(labels->key, OP_EQ, "Hello");
tt_str_op(labels->value, OP_EQ, "world");
tt_assert(labels->next);
tt_str_op(labels->next->key, OP_EQ, "This");
tt_str_op(labels->next->value, OP_EQ, "is a test");
tt_assert(labels->next->next);
tt_str_op(labels->next->next->key, OP_EQ, "Yadda");
tt_str_op(labels->next->next->value, OP_EQ, "yadda.");
tt_assert(labels->next->next->next == NULL);
/* Try reading this time. */
sz = 0;
as_read = storage_dir_read_labeled(d, fname, &labels2, &sz);
tt_assert(as_read);
tt_u64_op(sz, OP_EQ, bodylen);
tt_mem_op(as_read, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
tt_assert(config_lines_eq(labels, labels2));
done:
storage_dir_free(d);
tor_free(dirname);
tor_free(inp);
tor_free(fname);
config_free_lines(labels);
config_free_lines(labels2);
tor_munmap_file(map);
tor_free(as_read);
}
#define ENT(name) \
{ #name, test_storagedir_ ## name, TT_FORK, NULL, NULL }
@ -265,6 +368,8 @@ struct testcase_t storagedir_tests[] = {
ENT(deletion),
ENT(full),
ENT(cleaning),
ENT(save_labeled),
ENT(read_labeled),
END_OF_TESTCASES
};