gossip_store: add flag for spam gossip, update to v10

This will be used to decouple internal use of gossip from what is
passed to gossip peers. Updates GOSSIP_STORE_VERION to 10.

Changelog-Changed: gossip_store updated to version 10.
This commit is contained in:
Alex Myers 2022-05-03 12:42:31 -05:00 committed by Rusty Russell
parent 669bca4a02
commit cbafc0fa33
9 changed files with 27 additions and 14 deletions

View file

@ -53,6 +53,7 @@ u8 *gossip_store_next(const tal_t *ctx,
int *gossip_store_fd,
u32 timestamp_min, u32 timestamp_max,
bool push_only,
bool with_spam,
size_t *off, size_t *end)
{
u8 *msg = NULL;
@ -60,7 +61,7 @@ u8 *gossip_store_next(const tal_t *ctx,
while (!msg) {
struct gossip_hdr hdr;
u32 msglen, checksum, timestamp;
bool push;
bool push, ratelimited;
int type, r;
r = pread(*gossip_store_fd, &hdr, sizeof(hdr), *off);
@ -69,6 +70,7 @@ u8 *gossip_store_next(const tal_t *ctx,
msglen = be32_to_cpu(hdr.len);
push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT);
ratelimited = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT);
msglen &= GOSSIP_STORE_LEN_MASK;
/* Skip any deleted entries. */
@ -115,6 +117,8 @@ u8 *gossip_store_next(const tal_t *ctx,
msg = tal_free(msg);
} else if (!push && push_only) {
msg = tal_free(msg);
} else if (!with_spam && ratelimited) {
msg = tal_free(msg);
}
}

View file

@ -11,7 +11,7 @@ struct gossip_rcvd_filter;
/**
* gossip_store -- On-disk storage related information
*/
#define GOSSIP_STORE_VERSION 9
#define GOSSIP_STORE_VERSION 10
/**
* Bit of length we use to mark a deleted record.
@ -23,9 +23,15 @@ struct gossip_rcvd_filter;
*/
#define GOSSIP_STORE_LEN_PUSH_BIT 0x40000000U
/**
* Bit of length used to define a rate-limited record (do not rebroadcast)
*/
#define GOSSIP_STORE_LEN_RATELIMIT_BIT 0x20000000U
/* Mask for extracting just the length part of len field */
#define GOSSIP_STORE_LEN_MASK \
(~(GOSSIP_STORE_LEN_PUSH_BIT | GOSSIP_STORE_LEN_DELETED_BIT))
(~(GOSSIP_STORE_LEN_PUSH_BIT | GOSSIP_STORE_LEN_DELETED_BIT | \
GOSSIP_STORE_LEN_RATELIMIT_BIT))
/**
* gossip_hdr -- On-disk format header.
@ -47,6 +53,7 @@ u8 *gossip_store_next(const tal_t *ctx,
int *gossip_store_fd,
u32 timestamp_min, u32 timestamp_max,
bool push_only,
bool with_spam,
size_t *off, size_t *end);
/**

View file

@ -31,7 +31,7 @@ void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id U
* setup_gossip_store_test via od -v -Anone -tx1 < /tmp/ltests-kaf30pn0/test_gossip_store_compact_noappend_1/lightning-2/regtest/gossip_store
*/
static u8 canned_map[] = {
0x09, 0x80, 0x00, 0x01, 0xbc, 0x09, 0x8b, 0x67, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00
0x0a, 0x80, 0x00, 0x01, 0xbc, 0x09, 0x8b, 0x67, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00
, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, 0x01, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00

View file

@ -375,6 +375,7 @@ static u8 *maybe_from_gossip_store(const tal_t *ctx, struct peer *peer)
return gossip_store_next(ctx, &peer->daemon->gossip_store_fd,
0, 0xFFFFFFFF,
true,
false,
&peer->gs.off,
&peer->daemon->gossip_store_end);
}
@ -391,6 +392,7 @@ again:
peer->gs.timestamp_min,
peer->gs.timestamp_max,
false,
false,
&peer->gs.off,
&peer->daemon->gossip_store_end);
/* Don't send back gossip they sent to us! */

View file

@ -9,7 +9,7 @@ import io
import struct
# These duplicate constants in lightning/common/gossip_store.h
GOSSIP_STORE_VERSION = 9
GOSSIP_STORE_VERSIONS = [0x09, 0x0a]
GOSSIP_STORE_LEN_DELETED_BIT = 0x80000000
GOSSIP_STORE_LEN_PUSH_BIT = 0x40000000
GOSSIP_STORE_LEN_MASK = (~(GOSSIP_STORE_LEN_PUSH_BIT
@ -171,9 +171,9 @@ class Gossmap(object):
self.nodes: Dict[GossmapNodeId, GossmapNode] = {}
self.channels: Dict[ShortChannelId, GossmapChannel] = {}
self._last_scid: Optional[str] = None
version = self.store_file.read(1)
if version[0] != GOSSIP_STORE_VERSION:
raise ValueError("Invalid gossip store version {}".format(int(version)))
version = self.store_file.read(1)[0]
if version not in GOSSIP_STORE_VERSIONS:
raise ValueError("Invalid gossip store version {}".format(version))
self.bytes_read = 1
self.refresh()

View file

@ -118,7 +118,7 @@ static u8 *mk_private_channelmsg(const tal_t *ctx,
/* The upgrade from version 7 is trivial */
static bool can_upgrade(u8 oldversion)
{
return oldversion == 7 || oldversion == 8;
return oldversion == 7 || oldversion == 8 || oldversion == 9;
}
static bool upgrade_field(u8 oldversion,

View file

@ -1127,7 +1127,7 @@ def test_gossip_store_load(node_factory):
"""Make sure we can read canned gossip store"""
l1 = node_factory.get_node(start=False)
with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f:
f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION
f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION
"000001b0" # len
"fea676e8" # csum
"5b8d9b44" # timestamp
@ -1159,7 +1159,7 @@ def test_gossip_store_load_announce_before_update(node_factory):
"""Make sure we can read canned gossip store with node_announce before update. This happens when a channel_update gets replaced, leaving node_announce before it"""
l1 = node_factory.get_node(start=False)
with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f:
f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION
f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION
"000001b0" # len
"fea676e8" # csum
"5b8d9b44" # timestamp
@ -1202,7 +1202,7 @@ def test_gossip_store_load_amount_truncated(node_factory):
"""Make sure we can read canned gossip store with truncated amount"""
l1 = node_factory.get_node(start=False, allow_broken_log=True)
with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f:
f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION
f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION
"000001b0" # len
"fea676e8" # csum
"5b8d9b44" # timestamp
@ -1669,7 +1669,7 @@ def test_gossip_store_load_no_channel_update(node_factory):
# A channel announcement with no channel_update.
with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f:
f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION
f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION
"000001b0" # len
"fea676e8" # csum
"5b8d9b44" # timestamp
@ -1696,7 +1696,7 @@ def test_gossip_store_load_no_channel_update(node_factory):
l1.rpc.call('dev-compact-gossip-store')
with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), "rb") as f:
assert bytearray(f.read()) == bytearray.fromhex("09")
assert bytearray(f.read()) == bytearray.fromhex("0a")
@pytest.mark.developer("gossip without DEVELOPER=1 is slow")