Replace std's unmaintained bench with criterion

Rather than using the std benchmark framework (which isn't
maintained and is unlikely to get any further maintenance), we swap
for criterion, which at least gets us a variable number of test
runs so our benchmarks don't take forever.

We also fix the RGS benchmark to pass now that the file in use is
stale compared to today's date.

.github/workflows/build.yml

@@ -141,7 +141,12 @@ jobs:
           cd ..
       - name: Run benchmarks on Rust ${{ matrix.toolchain }}
         run: |
-          RUSTC_BOOTSTRAP=1 cargo bench --features _bench_unstable
+          cd bench
+          RUSTFLAGS="--cfg=ldk_bench --cfg=require_route_graph_test" cargo bench
+      - name: Run benchmarks with hashbrown on Rust ${{ matrix.toolchain }}
+        run: |
+          cd bench
+          RUSTFLAGS="--cfg=ldk_bench --cfg=require_route_graph_test" cargo bench --features hashbrown
 
   check_commits:
     runs-on: ubuntu-latest

Cargo.toml

@@ -14,6 +14,7 @@ exclude = [
     "lightning-custom-message",
     "lightning-transaction-sync",
     "no-std-check",
+    "bench",
 ]
 
 # Our tests do actual crypto and lots of work, the tradeoff for -O2 is well
@@ -35,8 +36,3 @@ lto = "off"
 opt-level = 3
 lto = true
 panic = "abort"
-
-[profile.bench]
-opt-level = 3
-codegen-units = 1
-lto = true

bench/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "lightning-bench"
+version = "0.0.1"
+authors = ["Matt Corallo"]
+edition = "2018"
+
+[[bench]]
+name = "bench"
+harness = false
+
+[features]
+hashbrown = ["lightning/hashbrown"]
+
+[dependencies]
+lightning = { path = "../lightning", features = ["_test_utils", "criterion"] }
+lightning-persister = { path = "../lightning-persister", features = ["criterion"] }
+lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync", features = ["criterion"] }
+criterion = { version = "0.4", default-features = false }
+
+[profile.release]
+opt-level = 3
+codegen-units = 1
+lto = true
+panic = "abort"
+debug = true

bench/benches/bench.rs

@@ -0,0 +1,22 @@
+extern crate lightning;
+extern crate lightning_persister;
+
+extern crate criterion;
+
+use criterion::{criterion_group, criterion_main};
+
+criterion_group!(benches,
+	// Note that benches run in the order given here. Thus, they're sorted according to how likely
+	// developers are to be working on the specific code listed, then by runtime.
+	lightning::routing::router::benches::generate_routes_with_zero_penalty_scorer,
+	lightning::routing::router::benches::generate_mpp_routes_with_zero_penalty_scorer,
+	lightning::routing::router::benches::generate_routes_with_probabilistic_scorer,
+	lightning::routing::router::benches::generate_mpp_routes_with_probabilistic_scorer,
+	lightning::routing::router::benches::generate_large_mpp_routes_with_probabilistic_scorer,
+	lightning::sign::benches::bench_get_secure_random_bytes,
+	lightning::ln::channelmanager::bench::bench_sends,
+	lightning_persister::bench::bench_sends,
+	lightning_rapid_gossip_sync::bench::bench_reading_full_graph_from_file,
+	lightning::routing::gossip::benches::read_network_graph,
+	lightning::routing::gossip::benches::write_network_graph);
+criterion_main!(benches);

lightning-persister/Cargo.toml

@@ -13,9 +13,6 @@ edition = "2018"
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
-[features]
-_bench_unstable = ["lightning/_bench_unstable"]
-
 [dependencies]
 bitcoin = "0.29.0"
 lightning = { version = "0.0.115", path = "../lightning" }
@@ -24,5 +21,8 @@ libc = "0.2"
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["winbase"] }
 
+[target.'cfg(ldk_bench)'.dependencies]
+criterion = { version = "0.4", optional = true, default-features = false }
+
 [dev-dependencies]
 lightning = { version = "0.0.115", path = "../lightning", features = ["_test_utils"] }

lightning-persister/src/lib.rs

@@ -8,8 +8,7 @@
 
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 
-#![cfg_attr(all(test, feature = "_bench_unstable"), feature(test))]
-#[cfg(all(test, feature = "_bench_unstable"))] extern crate test;
+#[cfg(ldk_bench)] extern crate criterion;
 
 mod util;
 
@@ -91,13 +90,13 @@ impl FilesystemPersister {
 				continue;
 			}
 
-			let txid = Txid::from_hex(filename.split_at(64).0)
+			let txid: Txid = Txid::from_hex(filename.split_at(64).0)
 				.map_err(|_| std::io::Error::new(
 					std::io::ErrorKind::InvalidData,
 					"Invalid tx ID in filename",
 				))?;
 
-			let index = filename.split_at(65).1.parse()
+			let index: u16 = filename.split_at(65).1.parse()
 				.map_err(|_| std::io::Error::new(
 					std::io::ErrorKind::InvalidData,
 					"Invalid tx index in filename",
@@ -338,14 +337,16 @@ mod tests {
 	}
 }
 
-#[cfg(all(test, feature = "_bench_unstable"))]
+#[cfg(ldk_bench)]
+/// Benches
 pub mod bench {
-	use test::Bencher;
+	use criterion::Criterion;
 
-	#[bench]
-	fn bench_sends(bench: &mut Bencher) {
+	/// Bench!
+	pub fn bench_sends(bench: &mut Criterion) {
 		let persister_a = super::FilesystemPersister::new("bench_filesystem_persister_a".to_string());
 		let persister_b = super::FilesystemPersister::new("bench_filesystem_persister_b".to_string());
-		lightning::ln::channelmanager::bench::bench_two_sends(bench, persister_a, persister_b);
+		lightning::ln::channelmanager::bench::bench_two_sends(
+			bench, "bench_filesystem_persisted_sends", persister_a, persister_b);
 	}
 }

lightning-rapid-gossip-sync/Cargo.toml

@@ -13,11 +13,13 @@ Utility to process gossip routing data from Rapid Gossip Sync Server.
 default = ["std"]
 no-std = ["lightning/no-std"]
 std = ["lightning/std"]
-_bench_unstable = []
 
 [dependencies]
 lightning = { version = "0.0.115", path = "../lightning", default-features = false }
 bitcoin = { version = "0.29.0", default-features = false }
 
+[target.'cfg(ldk_bench)'.dependencies]
+criterion = { version = "0.4", optional = true, default-features = false }
+
 [dev-dependencies]
 lightning = { version = "0.0.115", path = "../lightning", features = ["_test_utils"] }

lightning-rapid-gossip-sync/src/lib.rs

@@ -64,10 +64,7 @@
 
 #![cfg_attr(all(not(feature = "std"), not(test)), no_std)]
 
-// Allow and import test features for benching
-#![cfg_attr(all(test, feature = "_bench_unstable"), feature(test))]
-#[cfg(all(test, feature = "_bench_unstable"))]
-extern crate test;
+#[cfg(ldk_bench)] extern crate criterion;
 
 #[cfg(not(feature = "std"))]
 extern crate alloc;
@@ -287,36 +284,42 @@ mod tests {
 	}
 }
 
-#[cfg(all(test, feature = "_bench_unstable"))]
+#[cfg(ldk_bench)]
+/// Benches
 pub mod bench {
-	use test::Bencher;
-
 	use bitcoin::Network;
 
-	use lightning::ln::msgs::DecodeError;
+	use criterion::Criterion;
+
+	use std::fs;
+
 	use lightning::routing::gossip::NetworkGraph;
 	use lightning::util::test_utils::TestLogger;
 
 	use crate::RapidGossipSync;
 
-	#[bench]
-	fn bench_reading_full_graph_from_file(b: &mut Bencher) {
+	/// Bench!
+	pub fn bench_reading_full_graph_from_file(b: &mut Criterion) {
 		let logger = TestLogger::new();
-		b.iter(|| {
+		b.bench_function("read_full_graph_from_rgs", |b| b.iter(|| {
 			let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
 			let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
-			let sync_result = rapid_sync.sync_network_graph_with_file_path("./res/full_graph.lngossip");
-			if let Err(crate::error::GraphSyncError::DecodeError(DecodeError::Io(io_error))) = &sync_result {
-				let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin\n\n{:?}", io_error);
-				#[cfg(not(require_route_graph_test))]
-				{
-					println!("{}", error_string);
-					return;
-				}
-				#[cfg(require_route_graph_test)]
-				panic!("{}", error_string);
-			}
-			assert!(sync_result.is_ok())
-		});
+			let mut file = match fs::read("../lightning-rapid-gossip-sync/res/full_graph.lngossip") {
+				Ok(f) => f,
+				Err(io_error) => {
+					let error_string = format!(
+						"Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin\n\n{:?}",
+						io_error);
+					#[cfg(not(require_route_graph_test))]
+					{
+						println!("{}", error_string);
+						return;
+					}
+					#[cfg(require_route_graph_test)]
+					panic!("{}", error_string);
+				},
+			};
+			rapid_sync.update_network_graph_no_std(&mut file, None).unwrap();
+		}));
 	}
 }

lightning/Cargo.toml

@@ -28,7 +28,6 @@ max_level_trace = []
 # Allow signing of local transactions that may have been revoked or will be revoked, for functional testing (e.g. justice tx handling).
 # This is unsafe to use in production because it may result in the counterparty publishing taking our funds.
 unsafe_revoked_tx_signing = []
-_bench_unstable = []
 # Override signing to not include randomness when generating signatures for test vectors.
 _test_vectors = []
 
@@ -59,5 +58,8 @@ version = "0.29.0"
 default-features = false
 features = ["bitcoinconsensus", "secp-recovery"]
 
+[target.'cfg(ldk_bench)'.dependencies]
+criterion = { version = "0.4", optional = true, default-features = false }
+
 [target.'cfg(taproot)'.dependencies]
 musig2 = { git = "https://github.com/arik-so/rust-musig2", rev = "27797d7" }

lightning/src/lib.rs

@@ -54,9 +54,6 @@
 
 #![cfg_attr(all(not(feature = "std"), not(test)), no_std)]
 
-#![cfg_attr(all(any(test, feature = "_test_utils"), feature = "_bench_unstable"), feature(test))]
-#[cfg(all(any(test, feature = "_test_utils"), feature = "_bench_unstable"))] extern crate test;
-
 #[cfg(not(any(feature = "std", feature = "no-std")))]
 compile_error!("at least one of the `std` or `no-std` features must be enabled");
 
@@ -74,6 +71,8 @@ extern crate core;
 
 #[cfg(not(feature = "std"))] extern crate core2;
 
+#[cfg(ldk_bench)] extern crate criterion;
+
 #[macro_use]
 pub mod util;
 pub mod chain;
@@ -177,7 +176,7 @@ mod prelude {
 	pub use alloc::string::ToString;
 }
 
-#[cfg(all(not(feature = "_bench_unstable"), feature = "backtrace", feature = "std", test))]
+#[cfg(all(not(ldk_bench), feature = "backtrace", feature = "std", test))]
 extern crate backtrace;
 
 mod sync;

lightning/src/ln/channelmanager.rs

@@ -9267,7 +9267,7 @@ mod tests {
 	}
 }
 
-#[cfg(all(any(test, feature = "_test_utils"), feature = "_bench_unstable"))]
+#[cfg(ldk_bench)]
 pub mod bench {
 	use crate::chain::Listen;
 	use crate::chain::chainmonitor::{ChainMonitor, Persist};
@@ -9287,7 +9287,7 @@ pub mod bench {
 
 	use crate::sync::{Arc, Mutex};
 
-	use test::Bencher;
+	use criterion::Criterion;
 
 	type Manager<'a, P> = ChannelManager<
 		&'a ChainMonitor<InMemorySigner, &'a test_utils::TestChainSource,
@@ -9308,13 +9308,11 @@ pub mod bench {
 		fn chain_monitor(&self) -> Option<&test_utils::TestChainMonitor> { None }
 	}
 
-	#[cfg(test)]
-	#[bench]
-	fn bench_sends(bench: &mut Bencher) {
-		bench_two_sends(bench, test_utils::TestPersister::new(), test_utils::TestPersister::new());
+	pub fn bench_sends(bench: &mut Criterion) {
+		bench_two_sends(bench, "bench_sends", test_utils::TestPersister::new(), test_utils::TestPersister::new());
 	}
 
-	pub fn bench_two_sends<P: Persist<InMemorySigner>>(bench: &mut Bencher, persister_a: P, persister_b: P) {
+	pub fn bench_two_sends<P: Persist<InMemorySigner>>(bench: &mut Criterion, bench_name: &str, persister_a: P, persister_b: P) {
 		// Do a simple benchmark of sending a payment back and forth between two nodes.
 		// Note that this is unrealistic as each payment send will require at least two fsync
 		// calls per node.
@@ -9470,9 +9468,9 @@ pub mod bench {
 			}
 		}
 
-		bench.iter(|| {
+		bench.bench_function(bench_name, |b| b.iter(|| {
 			send_payment!(node_a, node_b);
 			send_payment!(node_b, node_a);
-		});
+		}));
 	}
 }

lightning/src/ln/functional_test_utils.rs

@@ -1769,7 +1769,7 @@ macro_rules! get_route_and_payment_hash {
 }
 
 #[macro_export]
-#[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
+#[cfg(any(test, ldk_bench, feature = "_test_utils"))]
 macro_rules! expect_payment_claimable {
 	($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr) => {
 		expect_payment_claimable!($node, $expected_payment_hash, $expected_payment_secret, $expected_recv_value, None, $node.node.get_our_node_id())
@@ -1796,7 +1796,7 @@ macro_rules! expect_payment_claimable {
 }
 
 #[macro_export]
-#[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
+#[cfg(any(test, ldk_bench, feature = "_test_utils"))]
 macro_rules! expect_payment_claimed {
 	($node: expr, $expected_payment_hash: expr, $expected_recv_value: expr) => {
 		let events = $node.node.get_and_clear_pending_events();
@@ -1913,7 +1913,7 @@ macro_rules! expect_payment_forwarded {
 	}
 }
 
-#[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
+#[cfg(any(test, ldk_bench, feature = "_test_utils"))]
 pub fn expect_channel_pending_event<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, expected_counterparty_node_id: &PublicKey) {
 	let events = node.node.get_and_clear_pending_events();
 	assert_eq!(events.len(), 1);
@@ -1925,7 +1925,7 @@ pub fn expect_channel_pending_event<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>,
 	}
 }
 
-#[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
+#[cfg(any(test, ldk_bench, feature = "_test_utils"))]
 pub fn expect_channel_ready_event<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, expected_counterparty_node_id: &PublicKey) {
 	let events = node.node.get_and_clear_pending_events();
 	assert_eq!(events.len(), 1);

lightning/src/routing/gossip.rs

@@ -3388,31 +3388,28 @@ pub(crate) mod tests {
 	}
 }
 
-#[cfg(all(test, feature = "_bench_unstable"))]
-mod benches {
+#[cfg(ldk_bench)]
+pub mod benches {
 	use super::*;
-
-	use test::Bencher;
 	use std::io::Read;
+	use criterion::{black_box, Criterion};
 
-	#[bench]
-	fn read_network_graph(bench: &mut Bencher) {
+	pub fn read_network_graph(bench: &mut Criterion) {
 		let logger = crate::util::test_utils::TestLogger::new();
 		let mut d = crate::routing::router::bench_utils::get_route_file().unwrap();
 		let mut v = Vec::new();
 		d.read_to_end(&mut v).unwrap();
-		bench.iter(|| {
-			let _ = NetworkGraph::read(&mut std::io::Cursor::new(&v), &logger).unwrap();
-		});
+		bench.bench_function("read_network_graph", |b| b.iter(||
+			NetworkGraph::read(&mut std::io::Cursor::new(black_box(&v)), &logger).unwrap()
+		));
 	}
 
-	#[bench]
-	fn write_network_graph(bench: &mut Bencher) {
+	pub fn write_network_graph(bench: &mut Criterion) {
 		let logger = crate::util::test_utils::TestLogger::new();
 		let mut d = crate::routing::router::bench_utils::get_route_file().unwrap();
 		let net_graph = NetworkGraph::read(&mut d, &logger).unwrap();
-		bench.iter(|| {
-			let _ = net_graph.encode();
-		});
+		bench.bench_function("write_network_graph", |b| b.iter(||
+			black_box(&net_graph).encode()
+		));
 	}
 }

lightning/src/routing/router.rs

@@ -1015,7 +1015,7 @@ struct PathBuildingHop<'a> {
 	/// decrease as well. Thus, we have to explicitly track which nodes have been processed and
 	/// avoid processing them again.
 	was_processed: bool,
-	#[cfg(all(not(feature = "_bench_unstable"), any(test, fuzzing)))]
+	#[cfg(all(not(ldk_bench), any(test, fuzzing)))]
 	// In tests, we apply further sanity checks on cases where we skip nodes we already processed
 	// to ensure it is specifically in cases where the fee has gone down because of a decrease in
 	// value_contribution_msat, which requires tracking it here. See comments below where it is
@@ -1036,7 +1036,7 @@ impl<'a> core::fmt::Debug for PathBuildingHop<'a> {
 			.field("path_penalty_msat", &self.path_penalty_msat)
 			.field("path_htlc_minimum_msat", &self.path_htlc_minimum_msat)
 			.field("cltv_expiry_delta", &self.candidate.cltv_expiry_delta());
-		#[cfg(all(not(feature = "_bench_unstable"), any(test, fuzzing)))]
+		#[cfg(all(not(ldk_bench), any(test, fuzzing)))]
 		let debug_struct = debug_struct
 			.field("value_contribution_msat", &self.value_contribution_msat);
 		debug_struct.finish()
@@ -1570,14 +1570,14 @@ where L::Target: Logger {
 								path_htlc_minimum_msat,
 								path_penalty_msat: u64::max_value(),
 								was_processed: false,
-								#[cfg(all(not(feature = "_bench_unstable"), any(test, fuzzing)))]
+								#[cfg(all(not(ldk_bench), any(test, fuzzing)))]
 								value_contribution_msat,
 							}
 						});
 
 						#[allow(unused_mut)] // We only use the mut in cfg(test)
 						let mut should_process = !old_entry.was_processed;
-						#[cfg(all(not(feature = "_bench_unstable"), any(test, fuzzing)))]
+						#[cfg(all(not(ldk_bench), any(test, fuzzing)))]
 						{
 							// In test/fuzzing builds, we do extra checks to make sure the skipping
 							// of already-seen nodes only happens in cases we expect (see below).
@@ -1648,13 +1648,13 @@ where L::Target: Logger {
 								old_entry.fee_msat = 0; // This value will be later filled with hop_use_fee_msat of the following channel
 								old_entry.path_htlc_minimum_msat = path_htlc_minimum_msat;
 								old_entry.path_penalty_msat = path_penalty_msat;
-								#[cfg(all(not(feature = "_bench_unstable"), any(test, fuzzing)))]
+								#[cfg(all(not(ldk_bench), any(test, fuzzing)))]
 								{
 									old_entry.value_contribution_msat = value_contribution_msat;
 								}
 								did_add_update_path_to_src_node = true;
 							} else if old_entry.was_processed && new_cost < old_cost {
-								#[cfg(all(not(feature = "_bench_unstable"), any(test, fuzzing)))]
+								#[cfg(all(not(ldk_bench), any(test, fuzzing)))]
 								{
 									// If we're skipping processing a node which was previously
 									// processed even though we found another path to it with a
@@ -6038,7 +6038,7 @@ mod tests {
 	}
 }
 
-#[cfg(all(test, not(feature = "no-std")))]
+#[cfg(all(any(test, ldk_bench), not(feature = "no-std")))]
 pub(crate) mod bench_utils {
 	use super::*;
 	use std::fs::File;
@@ -6068,7 +6068,18 @@ pub(crate) mod bench_utils {
 				path.pop(); // target
 				path.push("lightning");
 				path.push("net_graph-2023-01-18.bin");
-				eprintln!("{}", path.to_str().unwrap());
+				File::open(path)
+			})
+			.or_else(|_| { // Fall back to guessing based on the binary location for a subcrate
+				// path is likely something like .../rust-lightning/bench/target/debug/deps/bench..
+				let mut path = std::env::current_exe().unwrap();
+				path.pop(); // bench...
+				path.pop(); // deps
+				path.pop(); // debug
+				path.pop(); // target
+				path.pop(); // bench
+				path.push("lightning");
+				path.push("net_graph-2023-01-18.bin");
 				File::open(path)
 			})
 		.map_err(|_| "Please fetch https://bitcoin.ninja/ldk-net_graph-v0.0.113-2023-01-18.bin and place it at lightning/net_graph-2023-01-18.bin");
@@ -6204,8 +6215,8 @@ pub(crate) mod bench_utils {
 	}
 }
 
-#[cfg(all(test, feature = "_bench_unstable", not(feature = "no-std")))]
-mod benches {
+#[cfg(ldk_bench)]
+pub mod benches {
 	use super::*;
 	use crate::sign::{EntropySource, KeysManager};
 	use crate::ln::channelmanager;
@@ -6216,60 +6227,63 @@ mod benches {
 	use crate::util::logger::{Logger, Record};
 	use crate::util::test_utils::TestLogger;
 
-	use test::Bencher;
+	use criterion::Criterion;
 
 	struct DummyLogger {}
 	impl Logger for DummyLogger {
 		fn log(&self, _record: &Record) {}
 	}
 
-
-	#[bench]
-	fn generate_routes_with_zero_penalty_scorer(bench: &mut Bencher) {
+	pub fn generate_routes_with_zero_penalty_scorer(bench: &mut Criterion) {
 		let logger = TestLogger::new();
 		let network_graph = bench_utils::read_network_graph(&logger).unwrap();
 		let scorer = FixedPenaltyScorer::with_penalty(0);
-		generate_routes(bench, &network_graph, scorer, &(), InvoiceFeatures::empty(), 0);
+		generate_routes(bench, &network_graph, scorer, &(), InvoiceFeatures::empty(), 0,
+			"generate_routes_with_zero_penalty_scorer");
 	}
 
-	#[bench]
-	fn generate_mpp_routes_with_zero_penalty_scorer(bench: &mut Bencher) {
+	pub fn generate_mpp_routes_with_zero_penalty_scorer(bench: &mut Criterion) {
 		let logger = TestLogger::new();
 		let network_graph = bench_utils::read_network_graph(&logger).unwrap();
 		let scorer = FixedPenaltyScorer::with_penalty(0);
-		generate_routes(bench, &network_graph, scorer, &(), channelmanager::provided_invoice_features(&UserConfig::default()), 0);
+		generate_routes(bench, &network_graph, scorer, &(),
+			channelmanager::provided_invoice_features(&UserConfig::default()), 0,
+			"generate_mpp_routes_with_zero_penalty_scorer");
 	}
 
-	#[bench]
-	fn generate_routes_with_probabilistic_scorer(bench: &mut Bencher) {
+	pub fn generate_routes_with_probabilistic_scorer(bench: &mut Criterion) {
 		let logger = TestLogger::new();
 		let network_graph = bench_utils::read_network_graph(&logger).unwrap();
 		let params = ProbabilisticScoringFeeParameters::default();
 		let scorer = ProbabilisticScorer::new(ProbabilisticScoringDecayParameters::default(), &network_graph, &logger);
-		generate_routes(bench, &network_graph, scorer, &params, InvoiceFeatures::empty(), 0);
+		generate_routes(bench, &network_graph, scorer, &params, InvoiceFeatures::empty(), 0,
+			"generate_routes_with_probabilistic_scorer");
 	}
 
-	#[bench]
-	fn generate_mpp_routes_with_probabilistic_scorer(bench: &mut Bencher) {
+	pub fn generate_mpp_routes_with_probabilistic_scorer(bench: &mut Criterion) {
 		let logger = TestLogger::new();
 		let network_graph = bench_utils::read_network_graph(&logger).unwrap();
 		let params = ProbabilisticScoringFeeParameters::default();
 		let scorer = ProbabilisticScorer::new(ProbabilisticScoringDecayParameters::default(), &network_graph, &logger);
-		generate_routes(bench, &network_graph, scorer, &params, channelmanager::provided_invoice_features(&UserConfig::default()), 0);
+		generate_routes(bench, &network_graph, scorer, &params,
+			channelmanager::provided_invoice_features(&UserConfig::default()), 0,
+			"generate_mpp_routes_with_probabilistic_scorer");
 	}
 
-	#[bench]
-	fn generate_large_mpp_routes_with_probabilistic_scorer(bench: &mut Bencher) {
+	pub fn generate_large_mpp_routes_with_probabilistic_scorer(bench: &mut Criterion) {
 		let logger = TestLogger::new();
 		let network_graph = bench_utils::read_network_graph(&logger).unwrap();
 		let params = ProbabilisticScoringFeeParameters::default();
 		let scorer = ProbabilisticScorer::new(ProbabilisticScoringDecayParameters::default(), &network_graph, &logger);
-		generate_routes(bench, &network_graph, scorer, &params, channelmanager::provided_invoice_features(&UserConfig::default()), 100_000_000);
+		generate_routes(bench, &network_graph, scorer, &params,
+			channelmanager::provided_invoice_features(&UserConfig::default()), 100_000_000,
+			"generate_large_mpp_routes_with_probabilistic_scorer");
 	}
 
 	fn generate_routes<S: Score>(
-		bench: &mut Bencher, graph: &NetworkGraph<&TestLogger>, mut scorer: S,
+		bench: &mut Criterion, graph: &NetworkGraph<&TestLogger>, mut scorer: S,
 		score_params: &S::ScoreParams, features: InvoiceFeatures, starting_amount: u64,
+		bench_name: &'static str,
 	) {
 		let payer = bench_utils::payer_pubkey();
 		let keys_manager = KeysManager::new(&[0u8; 32], 42, 42);
@@ -6280,11 +6294,11 @@ mod benches {
 
 		// ...then benchmark finding paths between the nodes we learned.
 		let mut idx = 0;
-		bench.iter(|| {
+		bench.bench_function(bench_name, |b| b.iter(|| {
 			let (first_hop, params, amt) = &route_endpoints[idx % route_endpoints.len()];
 			assert!(get_route(&payer, params, &graph.read_only(), Some(&[first_hop]), *amt,
 				&DummyLogger{}, &scorer, score_params, &random_seed_bytes).is_ok());
 			idx += 1;
-		});
+		}));
 	}
 }

lightning/src/sign/mod.rs

@@ -1539,8 +1539,8 @@ pub fn dyn_sign() {
 	let _signer: Box<dyn EcdsaChannelSigner>;
 }
 
-#[cfg(all(test, feature = "_bench_unstable", not(feature = "no-std")))]
-mod benches {
+#[cfg(ldk_bench)]
+pub mod benches {
 	use std::sync::{Arc, mpsc};
 	use std::sync::mpsc::TryRecvError;
 	use std::thread;
@@ -1549,10 +1549,9 @@ mod benches {
 	use bitcoin::Network;
 	use crate::sign::{EntropySource, KeysManager};
 
-	use test::Bencher;
+	use criterion::Criterion;
 
-	#[bench]
-	fn bench_get_secure_random_bytes(bench: &mut Bencher) {
+	pub fn bench_get_secure_random_bytes(bench: &mut Criterion) {
 		let seed = [0u8; 32];
 		let now = Duration::from_secs(genesis_block(Network::Testnet).header.time as u64);
 		let keys_manager = Arc::new(KeysManager::new(&seed, now.as_secs(), now.subsec_micros()));
@@ -1578,11 +1577,8 @@ mod benches {
 			stops.push(stop_sender);
 		}
 
-		bench.iter(|| {
-			for _ in 1..100 {
-				keys_manager.get_secure_random_bytes();
-			}
-		});
+		bench.bench_function("get_secure_random_bytes", |b| b.iter(||
+			keys_manager.get_secure_random_bytes()));
 
 		for stop in stops {
 			let _ = stop.send(());
@@ -1591,5 +1587,4 @@ mod benches {
 			handle.join().unwrap();
 		}
 	}
-
 }

lightning/src/sync/mod.rs

@@ -3,7 +3,7 @@
 pub(crate) enum LockHeldState {
 	HeldByThread,
 	NotHeldByThread,
-	#[cfg(any(feature = "_bench_unstable", not(test)))]
+	#[cfg(any(ldk_bench, not(test)))]
 	Unsupported,
 }
 
@@ -20,20 +20,20 @@ pub(crate) trait LockTestExt<'a> {
 	fn unsafe_well_ordered_double_lock_self(&'a self) -> Self::ExclLock;
 }
 
-#[cfg(all(feature = "std", not(feature = "_bench_unstable"), test))]
+#[cfg(all(feature = "std", not(ldk_bench), test))]
 mod debug_sync;
-#[cfg(all(feature = "std", not(feature = "_bench_unstable"), test))]
+#[cfg(all(feature = "std", not(ldk_bench), test))]
 pub use debug_sync::*;
-#[cfg(all(feature = "std", not(feature = "_bench_unstable"), test))]
+#[cfg(all(feature = "std", not(ldk_bench), test))]
 // Note that to make debug_sync's regex work this must not contain `debug_string` in the module name
 mod test_lockorder_checks;
 
-#[cfg(all(feature = "std", any(feature = "_bench_unstable", not(test))))]
+#[cfg(all(feature = "std", any(ldk_bench, not(test))))]
 pub(crate) mod fairrwlock;
-#[cfg(all(feature = "std", any(feature = "_bench_unstable", not(test))))]
+#[cfg(all(feature = "std", any(ldk_bench, not(test))))]
 pub use {std::sync::{Arc, Mutex, Condvar, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}, fairrwlock::FairRwLock};
 
-#[cfg(all(feature = "std", any(feature = "_bench_unstable", not(test))))]
+#[cfg(all(feature = "std", any(ldk_bench, not(test))))]
 mod ext_impl {
 	use super::*;
 	impl<'a, T: 'a> LockTestExt<'a> for Mutex<T> {

lightning/src/util/test_utils.rs

@@ -738,7 +738,7 @@ impl Logger for TestLogger {
 	fn log(&self, record: &Record) {
 		*self.lines.lock().unwrap().entry((record.module_path.to_string(), format!("{}", record.args))).or_insert(0) += 1;
 		if record.level >= self.level {
-			#[cfg(feature = "std")]
+			#[cfg(all(not(ldk_bench), feature = "std"))]
 			println!("{:<5} {} [{} : {}, {}] {}", record.level.to_string(), self.id, record.module_path, record.file, record.line, record.args);
 		}
 	}