Write all statistics to disk exactly every 24 hours.

This commit is contained in:
Karsten Loesing 2009-08-19 15:41:12 +02:00
parent 10fbc998e1
commit 4e29f33427
9 changed files with 237 additions and 220 deletions

View File

@ -1060,24 +1060,26 @@ A filename containing GeoIP data, for use with BridgeRecordUsageByCountry.
.TP .TP
\fBCellStatistics \fR\fB0\fR|\fB1\fR\fP \fBCellStatistics \fR\fB0\fR|\fB1\fR\fP
When this option is enabled, Tor writes statistics on the mean time that When this option is enabled, Tor writes statistics on the mean time that
cells spend in circuit queues to disk every 24 hours. (Default: 0) cells spend in circuit queues to disk every 24 hours. Cannot be changed
while Tor is running. (Default: 0)
.LP .LP
.TP .TP
\fBDirReqStatistics \fR\fB0\fR|\fB1\fR\fP \fBDirReqStatistics \fR\fB0\fR|\fB1\fR\fP
When this option is enabled, Tor writes statistics on the number and When this option is enabled, Tor writes statistics on the number and
response time of network status requests to disk every 24 hours. response time of network status requests to disk every 24 hours. Cannot be
(Default: 0) changed while Tor is running. (Default: 0)
.LP .LP
.TP .TP
\fBEntryStatistics \fR\fB0\fR|\fB1\fR\fP \fBEntryStatistics \fR\fB0\fR|\fB1\fR\fP
When this option is enabled, Tor writes statistics on the number of When this option is enabled, Tor writes statistics on the number of
directly connecting clients to disk every 24 hours. (Default: 0) directly connecting clients to disk every 24 hours. Cannot be changed
while Tor is running. (Default: 0)
.LP .LP
.TP .TP
\fBExitPortStatistics \fR\fB0\fR|\fB1\fR\fP \fBExitPortStatistics \fR\fB0\fR|\fB1\fR\fP
When this option is enabled, Tor writes statistics on the number of When this option is enabled, Tor writes statistics on the number of
relayed bytes and opened stream per exit port to disk every 24 hours. relayed bytes and opened stream per exit port to disk every 24 hours.
(Default: 0) Cannot be changed while Tor is running. (Default: 0)
.LP .LP
.TP .TP
\fBExtraInfoStatistics \fR\fB0\fR|\fB1\fR\fP \fBExtraInfoStatistics \fR\fB0\fR|\fB1\fR\fP

View File

@ -449,7 +449,7 @@ circuit_free(circuit_t *circ)
or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); or_circuit_t *ocirc = TO_OR_CIRCUIT(circ);
/* Remember cell statistics for this circuit before deallocating. */ /* Remember cell statistics for this circuit before deallocating. */
if (get_options()->CellStatistics) if (get_options()->CellStatistics)
add_circ_to_buffer_stats(circ, time(NULL)); rep_hist_buffer_stats_add_circ(circ, time(NULL));
mem = ocirc; mem = ocirc;
memlen = sizeof(or_circuit_t); memlen = sizeof(or_circuit_t);
tor_assert(circ->magic == OR_CIRCUIT_MAGIC); tor_assert(circ->magic == OR_CIRCUIT_MAGIC);

View File

@ -1407,29 +1407,13 @@ options_act(or_options_t *old_options)
tor_free(actual_fname); tor_free(actual_fname);
} }
if (options->DirReqStatistics) { if (options->DirReqStatistics && !geoip_is_loaded()) {
/* Check if GeoIP database could be loaded. */ /* Check if GeoIP database could be loaded. */
if (!geoip_is_loaded()) { log_warn(LD_CONFIG, "Configured to measure directory request "
log_warn(LD_CONFIG, "Configured to measure directory request " "statistics, but no GeoIP database found!");
"statistics, but no GeoIP database found!"); return -1;
return -1;
}
log_notice(LD_CONFIG, "Configured to count directory requests by "
"country and write aggregate statistics to disk. Check the "
"dirreq-stats file in your data directory that will first "
"be written in 24 hours from now.");
} }
if (options->ExitPortStatistics)
log_notice(LD_CONFIG, "Configured to measure exit port statistics. "
"Look for the exit-stats file that will first be written to "
"the data directory in 24 hours from now.");
if (options->CellStatistics)
log_notice(LD_CONFIG, "Configured to measure cell statistics. Look "
"for the buffer-stats file that will first be written to "
"the data directory in 24 hours from now.");
if (options->EntryStatistics) { if (options->EntryStatistics) {
if (should_record_bridge_info(options)) { if (should_record_bridge_info(options)) {
/* Don't allow measuring statistics on entry guards when configured /* Don't allow measuring statistics on entry guards when configured
@ -1442,11 +1426,7 @@ options_act(or_options_t *old_options)
log_warn(LD_CONFIG, "Configured to measure entry node statistics, " log_warn(LD_CONFIG, "Configured to measure entry node statistics, "
"but no GeoIP database found!"); "but no GeoIP database found!");
return -1; return -1;
} else }
log_notice(LD_CONFIG, "Configured to measure entry node "
"statistics. Look for the entry-stats file that will "
"first be written to the data directory in 24 hours "
"from now.");
} }
/* Check if we need to parse and add the EntryNodes config option. */ /* Check if we need to parse and add the EntryNodes config option. */
@ -3784,6 +3764,16 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val,
return -1; return -1;
} }
if (old->CellStatistics != new_val->CellStatistics ||
old->DirReqStatistics != new_val->DirReqStatistics ||
old->EntryStatistics != new_val->EntryStatistics ||
old->ExitPortStatistics != new_val->ExitPortStatistics) {
*msg = tor_strdup("While Tor is running, changing either "
"CellStatistics, DirReqStatistics, EntryStatistics, "
"or ExitPortStatistics is not allowed.");
return -1;
}
return 0; return 0;
} }

View File

@ -1704,12 +1704,12 @@ connection_buckets_decrement(connection_t *conn, time_t now,
if (num_read > 0) { if (num_read > 0) {
if (conn->type == CONN_TYPE_EXIT) if (conn->type == CONN_TYPE_EXIT)
rep_hist_note_exit_bytes_read(conn->port, num_read, now); rep_hist_note_exit_bytes_read(conn->port, num_read);
rep_hist_note_bytes_read(num_read, now); rep_hist_note_bytes_read(num_read, now);
} }
if (num_written > 0) { if (num_written > 0) {
if (conn->type == CONN_TYPE_EXIT) if (conn->type == CONN_TYPE_EXIT)
rep_hist_note_exit_bytes_written(conn->port, num_written, now); rep_hist_note_exit_bytes_written(conn->port, num_written);
rep_hist_note_bytes_written(num_written, now); rep_hist_note_bytes_written(num_written, now);
} }

View File

@ -333,7 +333,7 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn)
escaped_safe_str(conn->address),conn->port, escaped_safe_str(conn->address),conn->port,
safe_str(fmt_addr(&conn->addr))); safe_str(fmt_addr(&conn->addr)));
rep_hist_note_exit_stream_opened(conn->port, approx_time()); rep_hist_note_exit_stream_opened(conn->port);
conn->state = EXIT_CONN_STATE_OPEN; conn->state = EXIT_CONN_STATE_OPEN;
connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */

View File

@ -12,8 +12,6 @@
#include "ht.h" #include "ht.h"
static void clear_geoip_db(void); static void clear_geoip_db(void);
static void dump_geoip_stats(void);
static void dump_entry_stats(void);
/** An entry from the GeoIP file: maps an IP range to a country. */ /** An entry from the GeoIP file: maps an IP range to a country. */
typedef struct geoip_entry_t { typedef struct geoip_entry_t {
@ -390,37 +388,6 @@ geoip_note_client_seen(geoip_client_action_t action,
return; return;
} }
/* Rotate the current request period. */
while (current_request_period_starts + REQUEST_HIST_PERIOD < now) {
if (!geoip_countries)
geoip_countries = smartlist_create();
if (!current_request_period_starts) {
current_request_period_starts = now;
break;
}
/* Also discard all items in the client history that are too old.
* (This only works here because bridge and directory stats are
* independent. Otherwise, we'd only want to discard those items
* with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */
geoip_remove_old_clients(current_request_period_starts);
/* Before rotating, write the current stats to disk. */
dump_geoip_stats();
if (get_options()->EntryStatistics)
dump_entry_stats();
/* Now rotate request period */
SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1],
sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1],
sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0;
c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0;
});
current_request_period_starts += REQUEST_HIST_PERIOD;
if (n_old_request_periods < REQUEST_HIST_LEN-1)
++n_old_request_periods;
}
lookup.ipaddr = addr; lookup.ipaddr = addr;
lookup.action = (int)action; lookup.action = (int)action;
ent = HT_FIND(clientmap, &client_history, &lookup); ent = HT_FIND(clientmap, &client_history, &lookup);
@ -940,12 +907,20 @@ geoip_get_request_history(time_t now, geoip_client_action_t action)
return result; return result;
} }
/** Store all our geoip statistics into $DATADIR/dirreq-stats. */ /** Start time of directory request stats. */
static void static time_t start_of_dirreq_stats_interval;
dump_geoip_stats(void)
/** Initialize directory request stats. */
void
geoip_dirreq_stats_init(time_t now)
{
start_of_dirreq_stats_interval = now;
}
/** Store all our geoip statistics into $DATADIR/dirreq-stats. */
void
geoip_dirreq_stats_write(time_t now)
{ {
time_t now = time(NULL);
time_t request_start;
char *filename = get_datadir_fname("dirreq-stats"); char *filename = get_datadir_fname("dirreq-stats");
char *data_v2 = NULL, *data_v3 = NULL; char *data_v2 = NULL, *data_v3 = NULL;
char written[ISO_TIME_LEN+1]; char written[ISO_TIME_LEN+1];
@ -957,11 +932,14 @@ dump_geoip_stats(void)
if (!get_options()->DirReqStatistics) if (!get_options()->DirReqStatistics)
goto done; goto done;
/* Discard all items in the client history that are too old. */
geoip_remove_old_clients(start_of_dirreq_stats_interval);
data_v2 = geoip_get_client_history_dirreq(now, data_v2 = geoip_get_client_history_dirreq(now,
GEOIP_CLIENT_NETWORKSTATUS_V2); GEOIP_CLIENT_NETWORKSTATUS_V2);
data_v3 = geoip_get_client_history_dirreq(now, data_v3 = geoip_get_client_history_dirreq(now,
GEOIP_CLIENT_NETWORKSTATUS); GEOIP_CLIENT_NETWORKSTATUS);
format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD); format_iso_time(written, now);
out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
0600, &open_file); 0600, &open_file);
if (!out) if (!out)
@ -973,8 +951,6 @@ dump_geoip_stats(void)
tor_free(data_v2); tor_free(data_v2);
tor_free(data_v3); tor_free(data_v3);
request_start = current_request_period_starts -
(n_old_request_periods * REQUEST_HIST_PERIOD);
data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2);
data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS);
if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n",
@ -1033,6 +1009,22 @@ dump_geoip_stats(void)
finish_writing_to_file(open_file); finish_writing_to_file(open_file);
open_file = NULL; open_file = NULL;
/* Rotate request period */
SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1],
sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1],
sizeof(uint32_t)*(REQUEST_HIST_LEN-1));
c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0;
c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0;
});
current_request_period_starts += REQUEST_HIST_PERIOD;
if (n_old_request_periods < REQUEST_HIST_LEN-1)
++n_old_request_periods;
start_of_dirreq_stats_interval = now;
done: done:
if (open_file) if (open_file)
abort_writing_to_file(open_file); abort_writing_to_file(open_file);
@ -1041,29 +1033,46 @@ dump_geoip_stats(void)
tor_free(data_v3); tor_free(data_v3);
} }
/** Start time of entry stats. */
static time_t start_of_entry_stats_interval;
/** Initialize entry stats. */
void
geoip_entry_stats_init(time_t now)
{
start_of_entry_stats_interval = now;
}
/** Store all our geoip statistics as entry guards into /** Store all our geoip statistics as entry guards into
* $DATADIR/entry-stats. */ * $DATADIR/entry-stats. */
static void void
dump_entry_stats(void) geoip_entry_stats_write(time_t now)
{ {
#ifdef ENABLE_ENTRY_STATS
time_t now = time(NULL);
char *filename = get_datadir_fname("entry-stats"); char *filename = get_datadir_fname("entry-stats");
char *data = NULL; char *data = NULL;
char written[ISO_TIME_LEN+1]; char written[ISO_TIME_LEN+1];
open_file_t *open_file = NULL; open_file_t *open_file = NULL;
FILE *out; FILE *out;
data = geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); if (!get_options()->EntryStatistics)
format_iso_time(written, geoip_get_history_start() + REQUEST_HIST_PERIOD); goto done;
/* Discard all items in the client history that are too old. */
geoip_remove_old_clients(start_of_entry_stats_interval);
data = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_CONNECT);
format_iso_time(written, now);
out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
0600, &open_file); 0600, &open_file);
if (!out) if (!out)
goto done; goto done;
if (fprintf(out, "entry-stats-end %s (%d s)\nentry-ips %s\n", if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n",
written, REQUEST_HIST_PERIOD, data ? data : "") < 0) written, (unsigned) (now - start_of_entry_stats_interval),
data ? data : "") < 0)
goto done; goto done;
start_of_entry_stats_interval = now;
finish_writing_to_file(open_file); finish_writing_to_file(open_file);
open_file = NULL; open_file = NULL;
done: done:
@ -1071,7 +1080,6 @@ dump_entry_stats(void)
abort_writing_to_file(open_file); abort_writing_to_file(open_file);
tor_free(filename); tor_free(filename);
tor_free(data); tor_free(data);
#endif
} }
/** Helper used to implement GETINFO ip-to-country/... controller command. */ /** Helper used to implement GETINFO ip-to-country/... controller command. */

View File

@ -830,7 +830,7 @@ run_scheduled_events(time_t now)
static time_t time_to_clean_caches = 0; static time_t time_to_clean_caches = 0;
static time_t time_to_recheck_bandwidth = 0; static time_t time_to_recheck_bandwidth = 0;
static time_t time_to_check_for_expired_networkstatus = 0; static time_t time_to_check_for_expired_networkstatus = 0;
static time_t time_to_dump_buffer_stats = 0; static time_t time_to_write_stats_files = 0;
static time_t time_to_retry_dns_init = 0; static time_t time_to_retry_dns_init = 0;
or_options_t *options = get_options(); or_options_t *options = get_options();
int i; int i;
@ -958,10 +958,44 @@ run_scheduled_events(time_t now)
time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL;
} }
if (time_to_dump_buffer_stats < now) { /* 1g. Check whether we should write statistics to disk.
if (get_options()->CellStatistics && time_to_dump_buffer_stats) */
dump_buffer_stats(); if (time_to_write_stats_files >= 0 && time_to_write_stats_files < now) {
time_to_dump_buffer_stats = now + DUMP_BUFFER_STATS_INTERVAL; #define WRITE_STATS_INTERVAL (24*60*60)
or_options_t *options = get_options();
if (options->CellStatistics || options->DirReqStatistics ||
options->EntryStatistics || options->ExitPortStatistics) {
if (!time_to_write_stats_files) {
/* Initialize stats. */
if (options->CellStatistics)
rep_hist_buffer_stats_init(now);
if (options->DirReqStatistics)
geoip_dirreq_stats_init(now);
if (options->EntryStatistics)
geoip_entry_stats_init(now);
if (options->ExitPortStatistics)
rep_hist_exit_stats_init(now);
log_notice(LD_CONFIG, "Configured to measure statistics. Look for "
"the *-stats files that will first be written to the "
"data directory in %d hours from now.",
WRITE_STATS_INTERVAL / (60 * 60));
time_to_write_stats_files = now + WRITE_STATS_INTERVAL;
} else {
/* Write stats to disk. */
time_to_write_stats_files += WRITE_STATS_INTERVAL;
if (options->CellStatistics)
rep_hist_buffer_stats_write(time_to_write_stats_files);
if (options->DirReqStatistics)
geoip_dirreq_stats_write(time_to_write_stats_files);
if (options->EntryStatistics)
geoip_entry_stats_write(time_to_write_stats_files);
if (options->ExitPortStatistics)
rep_hist_exit_stats_write(time_to_write_stats_files);
}
} else {
/* Never write stats to disk */
time_to_write_stats_files = -1;
}
} }
/* Remove old information from rephist and the rend cache. */ /* Remove old information from rephist and the rend cache. */

View File

@ -3739,6 +3739,11 @@ void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
dirreq_state_t new_state); dirreq_state_t new_state);
void geoip_dirreq_stats_init(time_t now);
void geoip_dirreq_stats_write(time_t now);
void geoip_entry_stats_init(time_t now);
void geoip_entry_stats_write(time_t now);
/********************************* hibernate.c **********************/ /********************************* hibernate.c **********************/
int accounting_parse_options(or_options_t *options, int validate_only); int accounting_parse_options(or_options_t *options, int validate_only);
@ -4076,11 +4081,11 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name);
void rep_hist_dump_stats(time_t now, int severity); void rep_hist_dump_stats(time_t now, int severity);
void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_read(size_t num_bytes, time_t when);
void rep_hist_note_bytes_written(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when);
void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes, void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes);
time_t now); void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes);
void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes, void rep_hist_note_exit_stream_opened(uint16_t port);
time_t now); void rep_hist_exit_stats_init(time_t now);
void rep_hist_note_exit_stream_opened(uint16_t port, time_t now); void rep_hist_exit_stats_write(time_t now);
int rep_hist_bandwidth_assess(void); int rep_hist_bandwidth_assess(void);
char *rep_hist_get_bandwidth_lines(int for_extrainfo); char *rep_hist_get_bandwidth_lines(int for_extrainfo);
void rep_hist_update_state(or_state_t *state); void rep_hist_update_state(or_state_t *state);
@ -4132,9 +4137,10 @@ void hs_usage_note_fetch_successful(const char *service_id, time_t now);
void hs_usage_write_statistics_to_file(time_t now); void hs_usage_write_statistics_to_file(time_t now);
void hs_usage_free_all(void); void hs_usage_free_all(void);
#define DUMP_BUFFER_STATS_INTERVAL (24*60*60) void rep_hist_buffer_stats_init(time_t now);
void add_circ_to_buffer_stats(circuit_t *circ, time_t end_of_interval); void rep_hist_buffer_stats_add_circ(circuit_t *circ,
void dump_buffer_stats(void); time_t end_of_interval);
void rep_hist_buffer_stats_write(time_t now);
/********************************* rendclient.c ***************************/ /********************************* rendclient.c ***************************/

View File

@ -1321,8 +1321,6 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when)
} }
/* Some constants */ /* Some constants */
/** How long are the intervals for measuring exit stats? */
#define EXIT_STATS_INTERVAL_SEC (24 * 60 * 60)
/** To what multiple should byte numbers be rounded up? */ /** To what multiple should byte numbers be rounded up? */
#define EXIT_STATS_ROUND_UP_BYTES 1024 #define EXIT_STATS_ROUND_UP_BYTES 1024
/** To what multiple should stream counts be rounded up? */ /** To what multiple should stream counts be rounded up? */
@ -1344,10 +1342,14 @@ static uint64_t *exit_bytes_written = NULL;
/** Number of streams opened in current period by exit port */ /** Number of streams opened in current period by exit port */
static uint32_t *exit_streams = NULL; static uint32_t *exit_streams = NULL;
/** Set up arrays for exit port statistics. */ /** When does the current exit stats period end? */
static void static time_t start_of_exit_stats_interval;
exit_stats_init(void)
/** Initialize exit port stats. */
void
rep_hist_exit_stats_init(time_t now)
{ {
start_of_exit_stats_interval = now;
exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS *
sizeof(uint64_t)); sizeof(uint64_t));
exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS *
@ -1356,12 +1358,9 @@ exit_stats_init(void)
sizeof(uint32_t)); sizeof(uint32_t));
} }
/** When does the current exit stats period end? */
static time_t end_of_current_exit_stats_period = 0;
/** Write exit stats for the current period to disk and reset counters. */ /** Write exit stats for the current period to disk and reset counters. */
static void void
write_exit_stats(time_t when) rep_hist_exit_stats_write(time_t now)
{ {
char t[ISO_TIME_LEN+1]; char t[ISO_TIME_LEN+1];
int r, i, comma; int r, i, comma;
@ -1372,98 +1371,93 @@ write_exit_stats(time_t when)
open_file_t *open_file = NULL; open_file_t *open_file = NULL;
FILE *out = NULL; FILE *out = NULL;
log_debug(LD_HIST, "Considering writing exit port statistics to disk.."); format_iso_time(t, now);
if (!exit_bytes_read) log_info(LD_HIST, "Writing exit port statistics to disk for period "
exit_stats_init(); "ending at %s.", t);
while (when > end_of_current_exit_stats_period) {
format_iso_time(t, end_of_current_exit_stats_period);
log_info(LD_HIST, "Writing exit port statistics to disk for period "
"ending at %s.", t);
if (!open_file) { if (!open_file) {
out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND,
0600, &open_file); 0600, &open_file);
if (!out) { if (!out) {
log_warn(LD_HIST, "Couldn't open '%s'.", filename); log_warn(LD_HIST, "Couldn't open '%s'.", filename);
goto done; goto done;
}
} }
}
/* written yyyy-mm-dd HH:MM:SS (n s) */ /* written yyyy-mm-dd HH:MM:SS (n s) */
if (fprintf(out, "exit-stats-end %s (%d s)\n", t, if (fprintf(out, "exit-stats-end %s (%d s)\n", t,
EXIT_STATS_INTERVAL_SEC) < 0) (unsigned) (now - start_of_exit_stats_interval)) < 0)
goto done;
/* Count the total number of bytes, so that we can attribute all
* observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL
* of all bytes to a special port 'other'. */
total_bytes = 0;
for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
total_bytes += exit_bytes_read[i];
total_bytes += exit_bytes_written[i];
}
threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL;
/* exit-kibibytes-(read|written) port=kibibytes,.. */
for (r = 0; r < 2; r++) {
b = r ? exit_bytes_read : exit_bytes_written;
tor_assert(b);
if (fprintf(out, "%s ",
r ? "exit-kibibytes-read"
: "exit-kibibytes-written") < 0)
goto done; goto done;
/* Count the total number of bytes, so that we can attribute all
* observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL
* of all bytes to a special port 'other'. */
total_bytes = 0;
for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
total_bytes += exit_bytes_read[i];
total_bytes += exit_bytes_written[i];
}
threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL;
/* kibibytes-(read|written) port=kibibytes,.. */
for (r = 0; r < 2; r++) {
b = r ? exit_bytes_read : exit_bytes_written;
tor_assert(b);
if (fprintf(out, "%s ",
r ? "exit-kibibytes-read"
: "exit-kibibytes-written") < 0)
goto done;
comma = 0;
other_bytes = 0;
for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
if (b[i] > 0) {
if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) {
uint64_t num = round_uint64_to_next_multiple_of(b[i],
EXIT_STATS_ROUND_UP_BYTES);
num /= 1024;
if (fprintf(out, "%s%d="U64_FORMAT,
comma++ ? "," : "", i,
U64_PRINTF_ARG(num)) < 0)
goto done;
} else
other_bytes += b[i];
}
}
other_bytes = round_uint64_to_next_multiple_of(other_bytes,
EXIT_STATS_ROUND_UP_BYTES);
other_bytes /= 1024;
if (fprintf(out, "%sother="U64_FORMAT"\n",
comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0)
goto done;
}
/* streams-opened port=num,.. */
if (fprintf(out, "exit-streams-opened ") < 0)
goto done;
comma = 0; comma = 0;
other_streams = 0; other_bytes = 0;
for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
if (exit_streams[i] > 0) { if (b[i] > 0) {
if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) {
uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], uint64_t num = round_uint64_to_next_multiple_of(b[i],
EXIT_STATS_ROUND_UP_STREAMS); EXIT_STATS_ROUND_UP_BYTES);
if (fprintf(out, "%s%d=%u", num /= 1024;
comma++ ? "," : "", i, num)<0) if (fprintf(out, "%s%d="U64_FORMAT,
comma++ ? "," : "", i,
U64_PRINTF_ARG(num)) < 0)
goto done; goto done;
} else } else
other_streams += exit_streams[i]; other_bytes += b[i];
} }
} }
other_streams = round_uint32_to_next_multiple_of(other_streams, other_bytes = round_uint64_to_next_multiple_of(other_bytes,
EXIT_STATS_ROUND_UP_STREAMS); EXIT_STATS_ROUND_UP_BYTES);
if (fprintf(out, "%sother=%u\n", other_bytes /= 1024;
comma ? "," : "", other_streams)<0) if (fprintf(out, "%sother="U64_FORMAT"\n",
comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0)
goto done; goto done;
/* Reset counters */
memset(exit_bytes_read, 0, sizeof(exit_bytes_read));
memset(exit_bytes_written, 0, sizeof(exit_bytes_written));
memset(exit_streams, 0, sizeof(exit_streams));
end_of_current_exit_stats_period += EXIT_STATS_INTERVAL_SEC;
} }
/* exit-streams-opened port=num,.. */
if (fprintf(out, "exit-streams-opened ") < 0)
goto done;
comma = 0;
other_streams = 0;
for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
if (exit_streams[i] > 0) {
if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) {
uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i],
EXIT_STATS_ROUND_UP_STREAMS);
if (fprintf(out, "%s%d=%u",
comma++ ? "," : "", i, num)<0)
goto done;
} else
other_streams += exit_streams[i];
}
}
other_streams = round_uint32_to_next_multiple_of(other_streams,
EXIT_STATS_ROUND_UP_STREAMS);
if (fprintf(out, "%sother=%u\n",
comma ? "," : "", other_streams)<0)
goto done;
/* Reset counters */
memset(exit_bytes_read, 0, sizeof(exit_bytes_read));
memset(exit_bytes_written, 0, sizeof(exit_bytes_written));
memset(exit_streams, 0, sizeof(exit_streams));
start_of_exit_stats_interval = now;
if (open_file) if (open_file)
finish_writing_to_file(open_file); finish_writing_to_file(open_file);
@ -1474,59 +1468,36 @@ write_exit_stats(time_t when)
tor_free(filename); tor_free(filename);
} }
/** Prepare to add an exit stats observation at second <b>when</b> by
* checking whether this observation lies in the current observation
* period; if not, shift the current period forward by one until the
* reported event fits it and write all results in between to disk. */
static void
add_exit_obs(time_t when)
{
if (!exit_bytes_read)
exit_stats_init();
if (when > end_of_current_exit_stats_period) {
if (end_of_current_exit_stats_period)
write_exit_stats(when);
else
end_of_current_exit_stats_period = when + EXIT_STATS_INTERVAL_SEC;
}
}
/** Note that we wrote <b>num_bytes</b> to an exit connection to /** Note that we wrote <b>num_bytes</b> to an exit connection to
* <b>port</b> in second <b>when</b>. */ * <b>port</b>. */
void void
rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes, rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes)
time_t when)
{ {
if (!get_options()->ExitPortStatistics) if (!get_options()->ExitPortStatistics)
return; return;
add_exit_obs(when);
exit_bytes_written[port] += num_bytes; exit_bytes_written[port] += num_bytes;
log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.",
(unsigned long)num_bytes, port); (unsigned long)num_bytes, port);
} }
/** Note that we read <b>num_bytes</b> from an exit connection to /** Note that we read <b>num_bytes</b> from an exit connection to
* <b>port</b> in second <b>when</b>. */ * <b>port</b>. */
void void
rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes, rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes)
time_t when)
{ {
if (!get_options()->ExitPortStatistics) if (!get_options()->ExitPortStatistics)
return; return;
add_exit_obs(when);
exit_bytes_read[port] += num_bytes; exit_bytes_read[port] += num_bytes;
log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.",
(unsigned long)num_bytes, port); (unsigned long)num_bytes, port);
} }
/** Note that we opened an exit stream to <b>port</b> in second /** Note that we opened an exit stream to <b>port</b>. */
* <b>when</b>. */
void void
rep_hist_note_exit_stream_opened(uint16_t port, time_t when) rep_hist_note_exit_stream_opened(uint16_t port)
{ {
if (!get_options()->ExitPortStatistics) if (!get_options()->ExitPortStatistics)
return; return;
add_exit_obs(when);
exit_streams[port]++; exit_streams[port]++;
log_debug(LD_HIST, "Opened exit stream to port %d", port); log_debug(LD_HIST, "Opened exit stream to port %d", port);
} }
@ -2623,7 +2594,14 @@ hs_usage_write_statistics_to_file(time_t now)
/*** cell statistics ***/ /*** cell statistics ***/
/** Start of the current buffer stats interval. */ /** Start of the current buffer stats interval. */
time_t start_of_buffer_stats_interval; static time_t start_of_buffer_stats_interval;
/** Initialize buffer stats. */
void
rep_hist_buffer_stats_init(time_t now)
{
start_of_buffer_stats_interval = now;
}
typedef struct circ_buffer_stats_t { typedef struct circ_buffer_stats_t {
uint32_t processed_cells; uint32_t processed_cells;
@ -2639,7 +2617,7 @@ smartlist_t *circuits_for_buffer_stats = NULL;
* <b>end_of_interval</b> and reset cell counters in case the circuit * <b>end_of_interval</b> and reset cell counters in case the circuit
* remains open in the next measurement interval. */ * remains open in the next measurement interval. */
void void
add_circ_to_buffer_stats(circuit_t *circ, time_t end_of_interval) rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
{ {
circ_buffer_stats_t *stat; circ_buffer_stats_t *stat;
time_t start_of_interval; time_t start_of_interval;
@ -2687,9 +2665,8 @@ _buffer_stats_compare_entries(const void **_a, const void **_b)
/** Append buffer statistics to local file. */ /** Append buffer statistics to local file. */
void void
dump_buffer_stats(void) rep_hist_buffer_stats_write(time_t now)
{ {
time_t now = time(NULL);
char *filename; char *filename;
char written[ISO_TIME_LEN+1]; char written[ISO_TIME_LEN+1];
open_file_t *open_file = NULL; open_file_t *open_file = NULL;
@ -2704,7 +2681,7 @@ dump_buffer_stats(void)
circuit_t *circ; circuit_t *circ;
/* add current circuits to stats */ /* add current circuits to stats */
for (circ = _circuit_get_global_list(); circ; circ = circ->next) for (circ = _circuit_get_global_list(); circ; circ = circ->next)
add_circ_to_buffer_stats(circ, now); rep_hist_buffer_stats_add_circ(circ, now);
/* calculate deciles */ /* calculate deciles */
memset(processed_cells, 0, SHARES * sizeof(int)); memset(processed_cells, 0, SHARES * sizeof(int));
memset(circs_in_share, 0, SHARES * sizeof(int)); memset(circs_in_share, 0, SHARES * sizeof(int));
@ -2736,7 +2713,7 @@ dump_buffer_stats(void)
goto done; goto done;
format_iso_time(written, now); format_iso_time(written, now);
if (fprintf(out, "cell-stats-end %s (%d s)\n", written, if (fprintf(out, "cell-stats-end %s (%d s)\n", written,
DUMP_BUFFER_STATS_INTERVAL) < 0) (unsigned) (now - start_of_buffer_stats_interval)) < 0)
goto done; goto done;
for (i = 0; i < SHARES; i++) { for (i = 0; i < SHARES; i++) {
tor_snprintf(buf, sizeof(buf), "%d", !circs_in_share[i] ? 0 : tor_snprintf(buf, sizeof(buf), "%d", !circs_in_share[i] ? 0 :