Introduce graph sync crate for fast-forwarding through gossip data downloaded from a server.

Author: arik-so

Diff for: .editorconfig

@@ -3,3 +3,4 @@
 [*]
 indent_style = tab
 insert_final_newline = true
+trim_trailing_whitespace = true

Diff for: .github/workflows/build.yml

@@ -141,6 +141,7 @@ jobs:
         run: |
           cargo test --verbose --color always  -p lightning
           cargo test --verbose --color always  -p lightning-invoice
+          cargo test --verbose --color always  -p lightning-rapid-gossip-sync
           cargo build --verbose  --color always -p lightning-persister
           cargo build --verbose  --color always -p lightning-background-processor
       - name: Test C Bindings Modifications on Rust ${{ matrix.toolchain }}
@@ -221,11 +222,24 @@ jobs:
       - name: Fetch routing graph snapshot
         if: steps.cache-graph.outputs.cache-hit != 'true'
         run: |
-          wget -O lightning/net_graph-2021-05-31.bin https://bitcoin.ninja/ldk-net_graph-v0.0.15-2021-05-31.bin
-          if [ "$(sha256sum lightning/net_graph-2021-05-31.bin | awk '{ print $1 }')" != "05a5361278f68ee2afd086cc04a1f927a63924be451f3221d380533acfacc303" ]; then
+          curl --verbose -L -o lightning/net_graph-2021-05-31.bin https://bitcoin.ninja/ldk-net_graph-v0.0.15-2021-05-31.bin
+          echo "Sha sum: $(sha256sum lightning/net_graph-2021-05-31.bin | awk '{ print $1 }')"
+          if [ "$(sha256sum lightning/net_graph-2021-05-31.bin | awk '{ print $1 }')" != "${EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM}" ]; then
             echo "Bad hash"
             exit 1
           fi
+        env:
+          EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM: 05a5361278f68ee2afd086cc04a1f927a63924be451f3221d380533acfacc303
+      - name: Fetch rapid graph sync reference input
+        run: |
+          curl --verbose -L -o lightning-rapid-gossip-sync/res/full_graph.lngossip https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin
+          echo "Sha sum: $(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')"
+          if [ "$(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')" != "${EXPECTED_RAPID_GOSSIP_SHASUM}" ]; then
+            echo "Bad hash"
+            exit 1
+          fi
+        env:
+          EXPECTED_RAPID_GOSSIP_SHASUM: 9637b91cea9d64320cf48fc0787c70fe69fc062f90d3512e207044110cadfd7b
       - name: Test with Network Graph on Rust ${{ matrix.toolchain }}
         run: |
           cd lightning

Diff for: Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "lightning-net-tokio",
     "lightning-persister",
     "lightning-background-processor",
+    "lightning-rapid-gossip-sync"
 ]
 
 exclude = [

Diff for: fuzz/Cargo.toml

@@ -19,6 +19,7 @@ stdin_fuzz = []
 [dependencies]
 afl = { version = "0.4", optional = true }
 lightning = { path = "../lightning", features = ["regex"] }
+lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync" }
 bitcoin = { version = "0.28.1", features = ["secp-lowmemory"] }
 hex = "0.3"
 honggfuzz = { version = "0.5", optional = true }

Diff for: fuzz/src/bin/gen_target.sh

@@ -10,6 +10,7 @@ GEN_TEST chanmon_deser
 GEN_TEST chanmon_consistency
 GEN_TEST full_stack
 GEN_TEST peer_crypt
+GEN_TEST process_network_graph
 GEN_TEST router
 GEN_TEST zbase32
 

Diff for: fuzz/src/bin/process_network_graph_target.rs

@@ -0,0 +1,113 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+// This file is auto-generated by gen_target.sh based on target_template.txt
+// To modify it, modify target_template.txt and run gen_target.sh instead.
+
+#![cfg_attr(feature = "libfuzzer_fuzz", no_main)]
+
+#[cfg(not(fuzzing))]
+compile_error!("Fuzz targets need cfg=fuzzing");
+
+extern crate lightning_fuzz;
+use lightning_fuzz::process_network_graph::*;
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+	fuzz!(|data| {
+		process_network_graph_run(data.as_ptr(), data.len());
+	});
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+	loop {
+		fuzz!(|data| {
+			process_network_graph_run(data.as_ptr(), data.len());
+		});
+	}
+}
+
+#[cfg(feature = "libfuzzer_fuzz")]
+#[macro_use] extern crate libfuzzer_sys;
+#[cfg(feature = "libfuzzer_fuzz")]
+fuzz_target!(|data: &[u8]| {
+	process_network_graph_run(data.as_ptr(), data.len());
+});
+
+#[cfg(feature = "stdin_fuzz")]
+fn main() {
+	use std::io::Read;
+
+	let mut data = Vec::with_capacity(8192);
+	std::io::stdin().read_to_end(&mut data).unwrap();
+	process_network_graph_run(data.as_ptr(), data.len());
+}
+
+#[test]
+fn run_test_cases() {
+	use std::fs;
+	use std::io::Read;
+	use lightning_fuzz::utils::test_logger::StringBuffer;
+
+	use std::sync::{atomic, Arc};
+	{
+		let data: Vec<u8> = vec![0];
+		process_network_graph_run(data.as_ptr(), data.len());
+	}
+	let mut threads = Vec::new();
+	let threads_running = Arc::new(atomic::AtomicUsize::new(0));
+	if let Ok(tests) = fs::read_dir("test_cases/process_network_graph") {
+		for test in tests {
+			let mut data: Vec<u8> = Vec::new();
+			let path = test.unwrap().path();
+			fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap();
+			threads_running.fetch_add(1, atomic::Ordering::AcqRel);
+
+			let thread_count_ref = Arc::clone(&threads_running);
+			let main_thread_ref = std::thread::current();
+			threads.push((path.file_name().unwrap().to_str().unwrap().to_string(),
+				std::thread::spawn(move || {
+					let string_logger = StringBuffer::new();
+
+					let panic_logger = string_logger.clone();
+					let res = if ::std::panic::catch_unwind(move || {
+						process_network_graph_test(&data, panic_logger);
+					}).is_err() {
+						Some(string_logger.into_string())
+					} else { None };
+					thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel);
+					main_thread_ref.unpark();
+					res
+				})
+			));
+			while threads_running.load(atomic::Ordering::Acquire) > 32 {
+				std::thread::park();
+			}
+		}
+	}
+	let mut failed_outputs = Vec::new();
+	for (test, thread) in threads.drain(..) {
+		if let Some(output) = thread.join().unwrap() {
+			println!("\nOutput of {}:\n{}\n", test, output);
+			failed_outputs.push(test);
+		}
+	}
+	if !failed_outputs.is_empty() {
+		println!("Test cases which failed: ");
+		for case in failed_outputs {
+			println!("{}", case);
+		}
+		panic!();
+	}
+}

Diff for: fuzz/src/lib.rs

@@ -9,6 +9,7 @@
 
 extern crate bitcoin;
 extern crate lightning;
+extern crate lightning_rapid_gossip_sync;
 extern crate hex;
 
 pub mod utils;
@@ -17,6 +18,7 @@ pub mod chanmon_deser;
 pub mod chanmon_consistency;
 pub mod full_stack;
 pub mod peer_crypt;
+pub mod process_network_graph;
 pub mod router;
 pub mod zbase32;
 

Diff for: fuzz/src/process_network_graph.rs

@@ -0,0 +1,20 @@
+// Import that needs to be added manually
+use utils::test_logger;
+
+/// Actual fuzz test, method signature and name are fixed
+fn do_test(data: &[u8]) {
+	let block_hash = bitcoin::BlockHash::default();
+	let network_graph = lightning::routing::network_graph::NetworkGraph::new(block_hash);
+	lightning_rapid_gossip_sync::processing::update_network_graph(&network_graph, data);
+}
+
+/// Method that needs to be added manually, {name}_test
+pub fn process_network_graph_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
+	do_test(data);
+}
+
+/// Method that needs to be added manually, {name}_run
+#[no_mangle]
+pub extern "C" fn process_network_graph_run(data: *const u8, datalen: usize) {
+	do_test(unsafe { std::slice::from_raw_parts(data, datalen) });
+}

Diff for: fuzz/targets.h

@@ -3,6 +3,7 @@ void chanmon_deser_run(const unsigned char* data, size_t data_len);
 void chanmon_consistency_run(const unsigned char* data, size_t data_len);
 void full_stack_run(const unsigned char* data, size_t data_len);
 void peer_crypt_run(const unsigned char* data, size_t data_len);
+void process_network_graph_run(const unsigned char* data, size_t data_len);
 void router_run(const unsigned char* data, size_t data_len);
 void zbase32_run(const unsigned char* data, size_t data_len);
 void msg_accept_channel_run(const unsigned char* data, size_t data_len);

Diff for: lightning-rapid-gossip-sync/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "lightning-rapid-gossip-sync"
+version = "0.0.104"
+authors = ["Arik Sosman <[email protected]>"]
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/lightningdevkit/rust-lightning"
+edition = "2018"
+description = """
+Utility to process gossip routing data from Rapid Gossip Sync Server.
+"""
+
+[features]
+_bench_unstable = []
+
+[dependencies]
+lightning = { version = "0.0.106", path = "../lightning" }
+bitcoin = { version = "0.28.1", default-features = false }
+
+[dev-dependencies]
+lightning = { version = "0.0.106", path = "../lightning", features = ["_test_utils"] }

Diff for: lightning-rapid-gossip-sync/README.md

@@ -0,0 +1,120 @@
+# lightning-rapid-gossip-sync
+
+This crate exposes functionality for rapid gossip graph syncing, aimed primarily at mobile clients.
+Its server counterpart is the
+[rapid-gossip-sync-server](https://github.com/lightningdevkit/rapid-gossip-sync-server) repository.
+
+## Mechanism
+
+The (presumed) server sends a compressed gossip response containing gossip data. The gossip data is
+formatted compactly, omitting signatures and opportunistically incremental where previous channel
+updates are known.
+
+Essentially, the serialization structure is as follows:
+
+1. Fixed prefix bytes `76, 68, 75, 1` (the first three bytes are ASCII for `LDK`)
+    - The purpose of this prefix is to identify the serialization format, should other rapid gossip
+      sync formats arise in the future
+    - The fourth byte is the protocol version in case our format gets updated
+2. Chain hash (32 bytes)
+3. Latest seen timestamp (`u32`)
+4. An unsigned int indicating the number of node IDs to follow
+5. An array of compressed node ID pubkeys (all pubkeys are presumed to be standard
+   compressed 33-byte-serializations)
+6. An unsigned int indicating the number of channel announcement messages to follow
+7. An array of significantly stripped down customized channel announcements
+8. An unsigned int indicating the number of channel update messages to follow
+9. A series of default values used for non-incremental channel updates
+    - The values are defined as follows:
+        1. `default_cltv_expiry_delta`
+        2. `default_htlc_minimum_msat`
+        3. `default_fee_base_msat`
+        4. `default_fee_proportional_millionths`
+        5. `default_htlc_maximum_msat` (`u64`, and if the default is no maximum, `u64::MAX`)
+    - The defaults are calculated by the server based on the frequency among non-incremental
+      updates within a given delta set
+10. An array of customized channel updates
+
+You will also notice that `NodeAnnouncement` messages are omitted altogether as the node IDs are
+implicitly extracted from the channel announcements and updates.
+
+The data is then applied to the current network graph, artificially dated to the timestamp of the
+latest seen message less one week, be it an announcement or an update, from the server's
+perspective. The network graph should not be pruned until the graph sync completes.
+
+### Custom Channel Announcement
+
+To achieve compactness and avoid data repetition, we're sending a significantly stripped down
+version of the channel announcement message, which contains only the following data:
+
+1. `channel_features`: `u16` + `n`, where `n` is the number of bytes indicated by the first `u16`
+2. `short_channel_id`: `CompactSize` (incremental `CompactSize` deltas starting from 0)
+3. `node_id_1_index`: `CompactSize` (index of node id within the previously sent sequence)
+4. `node_id_2_index`: `CompactSize` (index of node id within the previously sent sequence)
+
+### Custom Channel Update
+
+For the purpose of rapid syncing, we have deviated from the channel update format specified in
+BOLT 7 significantly. Our custom channel updates are structured as follows:
+
+1. `short_channel_id`: `CompactSize` (incremental `CompactSize` deltas starting at 0)
+2. `custom_channel_flags`: `u8`
+3. `update_data`
+
+Specifically, our custom channel flags break down like this:
+
+| 128                 | 64 | 32 | 16 | 8 | 4 | 2                | 1         |
+|---------------------|----|----|----|---|---|------------------|-----------|
+| Incremental update? |    |    |    |   |   | Disable channel? | Direction |
+
+If the most significant bit is set to `1`, indicating an incremental update, the intermediate bit
+flags assume the following meaning:
+
+| 64                              | 32                              | 16                          | 8                                         | 4                               |
+|---------------------------------|---------------------------------|-----------------------------|-------------------------------------------|---------------------------------|
+| `cltv_expiry_delta` has changed | `htlc_minimum_msat` has changed | `fee_base_msat` has changed | `fee_proportional_millionths` has changed | `htlc_maximum_msat` has changed |
+
+If the most significant bit is set to `0`, the meaning is almost identical, except instead of a
+change, the flags now represent a deviation from the defaults sent at the beginning of the update
+sequence.
+
+In both cases, `update_data` only contains the fields that are indicated by the channel flags to be
+non-default or to have mutated.
+
+## Delta Calculation
+
+The way a server is meant to calculate this rapid gossip sync data is by taking the latest time
+any change, be it either an announcement or an update, was seen. That timestamp is included in each
+rapid sync message, so all the client needs to do is cache one variable.
+
+If a particular channel update had never occurred before, the full update is sent. If a channel has
+had updates prior to the provided timestamp, the latest update prior to the timestamp is taken as a
+reference, and the delta is calculated against it.
+
+Depending on whether the rapid sync message is calculated on the fly or a snapshotted version is
+returned, intermediate changes between the latest update seen by the client and the latest update
+broadcast on the network may be taken into account when calculating the delta.
+
+## Performance
+
+Given the primary purpose of this utility is a faster graph sync, we thought it might be helpful to
+provide some examples of various delta sets. These examples were calculated as of May 19th  2022
+with a network graph comprised of 80,000 channel announcements and 160,000 directed channel updates.
+
+| Full sync                   |        |
+|-----------------------------|--------|
+| Message Length              | 4.7 MB |
+| Gzipped Message Length      | 2.0 MB |
+| Client-side Processing Time | 1.4 s  |
+
+| Week-old sync               |        |
+|-----------------------------|--------|
+| Message Length              | 2.7 MB |
+| Gzipped Message Length      | 862 kB |
+| Client-side Processing Time | 907 ms |
+
+| Day-old sync                |         |
+|-----------------------------|---------|
+| Message Length              | 191 kB  |
+| Gzipped Message Length      | 92.8 kB |
+| Client-side Processing Time | 196 ms  |

Diff for: lightning-rapid-gossip-sync/res/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore