Merge pull request #30 from TheBlueMatt/master

Working initial routing sync from lnd node
This commit is contained in:
Matt Corallo 2018-06-17 00:09:21 -04:00 committed by GitHub
commit d8474c9d3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 275 additions and 133 deletions

View file

@ -43,6 +43,14 @@ name = "full_stack_target"
path = "fuzz_targets/full_stack_target.rs"
# message fuzz targets
[[bin]]
name = "msg_ping_target"
path = "fuzz_targets/msg_ping_target.rs"
[[bin]]
name = "msg_pong_target"
path = "fuzz_targets/msg_pong_target.rs"
[[bin]]
name = "msg_accept_channel_target"
path = "fuzz_targets/msg_targets/msg_accept_channel_target.rs"

View file

@ -0,0 +1,63 @@
// This file is auto-generated by gen_target.sh based on msg_target_template.txt
// To modify it, modify msg_target_template.txt and run gen_target.sh instead.
extern crate lightning;
use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable, Ping};
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
if let Ok(msg) = Ping::decode(data) {
let _ = msg.encode();
}
}
#[cfg(feature = "afl")]
extern crate afl;
#[cfg(feature = "afl")]
fn main() {
afl::read_stdio_bytes(|data| {
do_test(&data);
});
}
#[cfg(feature = "honggfuzz")]
#[macro_use] extern crate honggfuzz;
#[cfg(feature = "honggfuzz")]
fn main() {
loop {
fuzz!(|data| {
do_test(data);
});
}
}
#[cfg(test)]
mod tests {
fn extend_vec_from_hex(hex: &str, out: &mut Vec<u8>) {
let mut b = 0;
for (idx, c) in hex.as_bytes().iter().enumerate() {
b <<= 4;
match *c {
b'A'...b'F' => b |= c - b'A' + 10,
b'a'...b'f' => b |= c - b'a' + 10,
b'0'...b'9' => b |= c - b'0',
_ => panic!("Bad hex"),
}
if (idx & 1) == 1 {
out.push(b);
b = 0;
}
}
}
#[test]
fn duplicate_crash() {
let mut a = Vec::new();
extend_vec_from_hex("00", &mut a);
super::do_test(&a);
}
}

View file

@ -0,0 +1,63 @@
// This file is auto-generated by gen_target.sh based on msg_target_template.txt
// To modify it, modify msg_target_template.txt and run gen_target.sh instead.
extern crate lightning;
use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable, Pong};
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
if let Ok(msg) = Pong::decode(data) {
let _ = msg.encode();
}
}
#[cfg(feature = "afl")]
extern crate afl;
#[cfg(feature = "afl")]
fn main() {
afl::read_stdio_bytes(|data| {
do_test(&data);
});
}
#[cfg(feature = "honggfuzz")]
#[macro_use] extern crate honggfuzz;
#[cfg(feature = "honggfuzz")]
fn main() {
loop {
fuzz!(|data| {
do_test(data);
});
}
}
#[cfg(test)]
mod tests {
fn extend_vec_from_hex(hex: &str, out: &mut Vec<u8>) {
let mut b = 0;
for (idx, c) in hex.as_bytes().iter().enumerate() {
b <<= 4;
match *c {
b'A'...b'F' => b |= c - b'A' + 10,
b'a'...b'f' => b |= c - b'a' + 10,
b'0'...b'9' => b |= c - b'0',
_ => panic!("Bad hex"),
}
if (idx & 1) == 1 {
out.push(b);
b = 0;
}
}
}
#[test]
fn duplicate_crash() {
let mut a = Vec::new();
extend_vec_from_hex("00", &mut a);
super::do_test(&a);
}
}

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::AcceptChannel, data, read_pos);
}
test_msg!(msgs::AcceptChannel, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::ClosingSigned, data, read_pos);
}
test_msg!(msgs::ClosingSigned, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::CommitmentSigned, data, read_pos);
}
test_msg!(msgs::CommitmentSigned, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::FundingCreated, data, read_pos);
}
test_msg!(msgs::FundingCreated, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::FundingLocked, data, read_pos);
}
test_msg!(msgs::FundingLocked, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::FundingSigned, data, read_pos);
}
test_msg!(msgs::FundingSigned, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::OpenChannel, data, read_pos);
}
test_msg!(msgs::OpenChannel, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::RevokeAndACK, data, read_pos);
}
test_msg!(msgs::RevokeAndACK, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::Shutdown, data, read_pos);
}
test_msg!(msgs::Shutdown, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::MSG_TARGET, data, read_pos);
}
test_msg!(msgs::MSG_TARGET, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::UpdateAddHTLC, data, read_pos);
}
test_msg!(msgs::UpdateAddHTLC, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::UpdateFailHTLC, data, read_pos);
}
test_msg!(msgs::UpdateFailHTLC, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::UpdateFailMalformedHTLC, data, read_pos);
}
test_msg!(msgs::UpdateFailMalformedHTLC, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::UpdateFee, data, read_pos);
}
test_msg!(msgs::UpdateFee, data);
}
#[cfg(feature = "afl")]

View file

@ -9,15 +9,11 @@ use lightning::util::reset_rng_state;
use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
mod utils;
use utils::slice_to_be16;
#[inline]
pub fn do_test(data: &[u8]) {
reset_rng_state();
let mut read_pos = 0;
loop {
test_msg!(msgs::UpdateFulfillHTLC, data, read_pos);
}
test_msg!(msgs::UpdateFulfillHTLC, data);
}
#[cfg(feature = "afl")]

View file

@ -1,54 +1,17 @@
#![macro_use]
#[allow(dead_code)]
#[inline]
pub fn slice_to_be16(v: &[u8]) -> u16 {
((v[0] as u16) << 8*1) |
((v[1] as u16) << 8*0)
}
#[macro_export]
macro_rules! test_msg {
($MsgType: path, $data: ident, $read_pos: ident) => {
($MsgType: path, $data: ident) => {
{
let len = slice_to_be16(get_slice!($data, $read_pos, 2));
let raw = get_slice!($data, $read_pos, len);
let cb = decode_msg!($MsgType, raw).encode();
assert_eq!(&raw[..cb.len()], &cb[..]);
}
}
}
#[macro_export]
macro_rules! decode_msg {
($MsgType: path, $data: expr) => {
match <($MsgType)>::decode($data) {
Ok(msg) => msg,
Err(e) => match e {
msgs::DecodeError::UnknownRealmByte => return,
msgs::DecodeError::BadPublicKey => return,
msgs::DecodeError::BadSignature => return,
msgs::DecodeError::ExtraAddressesPerType => return,
msgs::DecodeError::WrongLength => return,
if let Ok(msg) = <$MsgType as MsgDecodable>::decode($data){
let enc = msg.encode();
assert_eq!(&$data[..enc.len()], &enc[..]);
}
}
}
}
#[macro_export]
macro_rules! get_slice {
($data: ident, $read_pos: ident, $len: expr) => {
{
let slice_len = $len as usize;
if $data.len() < $read_pos + slice_len {
return;
}
$read_pos += slice_len;
&$data[$read_pos - slice_len..$read_pos]
}
}
}
#[allow(dead_code)]
#[cfg(test)]
pub fn extend_vec_from_hex(hex: &str, out: &mut Vec<u8>) {

View file

@ -61,9 +61,16 @@ impl LocalFeatures {
self.flags.len() > 0 && (self.flags[0] & 1) != 0
}
pub fn supports_initial_routing_sync(&self) -> bool {
pub fn initial_routing_sync(&self) -> bool {
self.flags.len() > 0 && (self.flags[0] & (1 << 3)) != 0
}
pub fn set_initial_routing_sync(&mut self) {
if self.flags.len() == 0 {
self.flags.resize(1, 1 << 3);
} else {
self.flags[0] |= 1 << 3;
}
}
pub fn supports_upfront_shutdown_script(&self) -> bool {
self.flags.len() > 0 && (self.flags[0] & (3 << 4)) != 0
@ -132,6 +139,15 @@ pub struct Init {
pub local_features: LocalFeatures,
}
pub struct Ping {
pub ponglen: u16,
pub byteslen: u16,
}
pub struct Pong {
pub byteslen: u16,
}
pub struct OpenChannel {
pub chain_hash: Sha256dHash,
pub temporary_channel_id: Uint256,
@ -357,7 +373,9 @@ pub enum ErrorAction {
msg: UpdateFailHTLC
},
/// The peer took some action which made us think they were useless. Disconnect them.
DisconnectPeer {},
DisconnectPeer,
/// The peer did something harmless that we weren't able to process, just log and ignore
IgnoreError,
}
pub struct HandleError { //TODO: rename me
@ -506,7 +524,7 @@ macro_rules! secp_signature {
impl MsgDecodable for LocalFeatures {
fn decode(v: &[u8]) -> Result<Self, DecodeError> {
if v.len() < 3 { return Err(DecodeError::WrongLength); }
if v.len() < 2 { return Err(DecodeError::WrongLength); }
let len = byte_utils::slice_to_be16(&v[0..2]) as usize;
if v.len() < len + 2 { return Err(DecodeError::WrongLength); }
let mut flags = Vec::with_capacity(len);
@ -528,7 +546,7 @@ impl MsgEncodable for LocalFeatures {
impl MsgDecodable for GlobalFeatures {
fn decode(v: &[u8]) -> Result<Self, DecodeError> {
if v.len() < 3 { return Err(DecodeError::WrongLength); }
if v.len() < 2 { return Err(DecodeError::WrongLength); }
let len = byte_utils::slice_to_be16(&v[0..2]) as usize;
if v.len() < len + 2 { return Err(DecodeError::WrongLength); }
let mut flags = Vec::with_capacity(len);
@ -570,6 +588,54 @@ impl MsgEncodable for Init {
}
}
impl MsgDecodable for Ping {
fn decode(v: &[u8]) -> Result<Self, DecodeError> {
if v.len() < 4 {
return Err(DecodeError::WrongLength);
}
let ponglen = byte_utils::slice_to_be16(&v[0..2]);
let byteslen = byte_utils::slice_to_be16(&v[2..4]);
if v.len() < 4 + byteslen as usize {
return Err(DecodeError::WrongLength);
}
Ok(Self {
ponglen,
byteslen,
})
}
}
impl MsgEncodable for Ping {
fn encode(&self) -> Vec<u8> {
let mut res = Vec::with_capacity(self.byteslen as usize + 2);
res.extend_from_slice(&byte_utils::be16_to_array(self.byteslen));
res.resize(2 + self.byteslen as usize, 0);
res
}
}
impl MsgDecodable for Pong {
fn decode(v: &[u8]) -> Result<Self, DecodeError> {
if v.len() < 2 {
return Err(DecodeError::WrongLength);
}
let byteslen = byte_utils::slice_to_be16(&v[0..2]);
if v.len() < 2 + byteslen as usize {
return Err(DecodeError::WrongLength);
}
Ok(Self {
byteslen
})
}
}
impl MsgEncodable for Pong {
fn encode(&self) -> Vec<u8> {
let mut res = Vec::with_capacity(self.byteslen as usize + 2);
res.extend_from_slice(&byte_utils::be16_to_array(self.byteslen));
res.resize(2 + self.byteslen as usize, 0);
res
}
}
impl MsgDecodable for OpenChannel {
fn decode(v: &[u8]) -> Result<Self, DecodeError> {
if v.len() < 2*32+6*8+4+2*2+6*33+1 {

View file

@ -8,6 +8,7 @@ use util::events::{EventsProvider,Event};
use std::collections::{HashMap,LinkedList};
use std::sync::{Arc, Mutex};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{cmp,error,mem,hash,fmt};
pub struct MessageHandler {
@ -86,6 +87,7 @@ pub struct PeerManager<Descriptor: SocketDescriptor> {
peers: Mutex<PeerHolder<Descriptor>>,
pending_events: Mutex<Vec<Event>>,
our_node_secret: SecretKey,
initial_syncs_sent: AtomicUsize,
}
@ -101,6 +103,9 @@ macro_rules! encode_msg {
}
}
//TODO: Really should do something smarter for this
const INITIAL_SYNCS_TO_SEND: usize = 5;
/// Manages and reacts to connection events. You probably want to use file descriptors as PeerIds.
/// PeerIds may repeat, but only after disconnect_event() has been called.
impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
@ -110,6 +115,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
peers: Mutex::new(PeerHolder { peers: HashMap::new(), node_id_to_descriptor: HashMap::new() }),
pending_events: Mutex::new(Vec::new()),
our_node_secret: our_node_secret,
initial_syncs_sent: AtomicUsize::new(0),
}
}
@ -272,16 +278,19 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
match $thing {
Ok(x) => x,
Err(e) => {
// TODO: Log e.err
println!("Got error handling message: {}!", e.err);
if let Some(action) = e.msg {
match action {
msgs::ErrorAction::UpdateFailHTLC { msg } => {
encode_and_send_msg!(msg, 131);
continue;
},
msgs::ErrorAction::DisconnectPeer {} => {
msgs::ErrorAction::DisconnectPeer => {
return Err(PeerHandleError{ no_connection_possible: false });
},
msgs::ErrorAction::IgnoreError => {
continue;
},
}
} else {
return Err(PeerHandleError{ no_connection_possible: false });
@ -296,6 +305,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
match $thing {
Ok(x) => x,
Err(_e) => {
println!("Error decoding message");
//TODO: Handle e?
return Err(PeerHandleError{ no_connection_possible: false });
}
@ -303,6 +313,18 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
}
}
macro_rules! try_ignore_potential_decodeerror {
($thing: expr) => {
match $thing {
Ok(x) => x,
Err(_e) => {
println!("Error decoding message, ignoring due to lnd spec incompatibility. See https://github.com/lightningnetwork/lnd/issues/1407");
continue;
}
};
}
}
let next_step = peer.channel_encryptor.get_noise_step();
match next_step {
NextNoiseStep::ActOne => {
@ -317,9 +339,14 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
peer.pending_read_is_header = true;
insert_node_id = Some(peer.their_node_id.unwrap());
let mut local_features = msgs::LocalFeatures::new();
if self.initial_syncs_sent.load(Ordering::Acquire) < INITIAL_SYNCS_TO_SEND {
self.initial_syncs_sent.fetch_add(1, Ordering::AcqRel);
local_features.set_initial_routing_sync();
}
encode_and_send_msg!(msgs::Init {
global_features: msgs::GlobalFeatures::new(),
local_features: msgs::LocalFeatures::new(),
local_features,
}, 16);
},
NextNoiseStep::ActThree => {
@ -365,17 +392,29 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
peer.their_local_features = Some(msg.local_features);
if !peer.outbound {
let mut local_features = msgs::LocalFeatures::new();
if self.initial_syncs_sent.load(Ordering::Acquire) < INITIAL_SYNCS_TO_SEND {
self.initial_syncs_sent.fetch_add(1, Ordering::AcqRel);
local_features.set_initial_routing_sync();
}
encode_and_send_msg!(msgs::Init {
global_features: msgs::GlobalFeatures::new(),
local_features: msgs::LocalFeatures::new(),
local_features,
}, 16);
}
},
17 => {
// Error msg
},
18 => { }, // ping
19 => { }, // pong
18 => {
let msg = try_potential_decodeerror!(msgs::Ping::decode(&msg_data[2..]));
let resp = msgs::Pong { byteslen: msg.ponglen };
encode_and_send_msg!(resp, 19);
},
19 => {
try_potential_decodeerror!(msgs::Pong::decode(&msg_data[2..]));
},
// Channel control:
32 => {
@ -491,7 +530,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
}
},
257 => {
let msg = try_potential_decodeerror!(msgs::NodeAnnouncement::decode(&msg_data[2..]));
let msg = try_ignore_potential_decodeerror!(msgs::NodeAnnouncement::decode(&msg_data[2..]));
try_potential_handleerror!(self.message_handler.route_handler.handle_node_announcement(&msg));
},
258 => {

View file

@ -3,7 +3,7 @@ use secp256k1::{Secp256k1,Message};
use bitcoin::util::hash::Sha256dHash;
use ln::msgs::{HandleError,RoutingMessageHandler,MsgEncodable,NetAddress,GlobalFeatures};
use ln::msgs::{ErrorAction,HandleError,RoutingMessageHandler,MsgEncodable,NetAddress,GlobalFeatures};
use ln::msgs;
use std::cmp;
@ -122,10 +122,10 @@ impl RoutingMessageHandler for Router {
let mut network = self.network_map.write().unwrap();
match network.nodes.get_mut(&msg.contents.node_id) {
None => Err(HandleError{err: "No existing channels for node_announcement", msg: None}),
None => Err(HandleError{err: "No existing channels for node_announcement", msg: Some(ErrorAction::IgnoreError)}),
Some(node) => {
if node.last_update >= msg.contents.timestamp {
return Err(HandleError{err: "Update older than last processed update", msg: None});
return Err(HandleError{err: "Update older than last processed update", msg: Some(ErrorAction::IgnoreError)});
}
node.features = msg.contents.features.clone();
@ -159,7 +159,7 @@ impl RoutingMessageHandler for Router {
//TODO: because asking the blockchain if short_channel_id is valid is only optional
//in the blockchain API, we need to handle it smartly here, though its unclear
//exactly how...
return Err(HandleError{err: "Already have knowledge of channel", msg: None})
return Err(HandleError{err: "Already have knowledge of channel", msg: Some(ErrorAction::IgnoreError)})
},
Entry::Vacant(entry) => {
entry.insert(ChannelInfo {
@ -233,12 +233,12 @@ impl RoutingMessageHandler for Router {
let chan_was_enabled;
match network.channels.get_mut(&NetworkMap::get_key(msg.contents.short_channel_id, msg.contents.chain_hash)) {
None => return Err(HandleError{err: "Couldn't find channel for update", msg: None}),
None => return Err(HandleError{err: "Couldn't find channel for update", msg: Some(ErrorAction::IgnoreError)}),
Some(channel) => {
macro_rules! maybe_update_channel_info {
( $target: expr) => {
if $target.last_update >= msg.contents.timestamp {
return Err(HandleError{err: "Update older than last processed update", msg: None});
return Err(HandleError{err: "Update older than last processed update", msg: Some(ErrorAction::IgnoreError)});
}
chan_was_enabled = $target.enabled;
$target.last_update = msg.contents.timestamp;