mirrors/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2024-11-19 18:09:47 +01:00
Code Issues Projects Releases Wiki Activity

clusterlin: include topological pot subsets automatically (optimization)

Browse Source 
Automatically add topologically-valid subsets of the potential set pot
to inc. It can be proven that these must be part of the best reachable
topologically-valid set from that work item.

This is a crucial optimization that (apparently) reduces the maximum
number of iterations from ~2^(N-1) to ~sqrt(2^N).

Co-Authored-By: Suhas Daftuar <sdaftuar@gmail.com>
This commit is contained in:
Pieter Wuille 2024-05-09 13:53:27 -04:00
parent e20fda77a2
commit 71f2629398
3 changed files with 57 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/bench/cluster_linearize.cpp
View File

@ -48,8 +48,8 @@ DepGraph<SetType> MakeWideGraph(ClusterIndex ntx)
return depgraph;
}
// Construct a difficult graph. These need at least sqrt(2^(n-1)) iterations in the best
// known algorithms (purely empirically determined).
// Construct a difficult graph. These need at least sqrt(2^(n-1)) iterations in the implemented
// algorithm (purely empirically determined).
template<typename SetType>
DepGraph<SetType> MakeHardGraph(ClusterIndex ntx)
{
@ -316,7 +316,14 @@ static constexpr auto BENCH_EXAMPLE_08 = "83728ce80000b90befca1001806083b24002b4
static constexpr auto BENCH_EXAMPLE_09 = "bf2989d00400815bca5c01af1e86f97602800d9d6c03800d8a3404b47988866e05b36287f92e0680109f68078010991a08805ecf1208076e80933e09078062d01c0a078054b6760b078053b6760c076f9c1c0d078054b6760e0771af260f0771b17e10078032f57011078035d56812078054e1581307886b83dc301407817480d13013068005a6001406803d80821a15066ef3201606800ea2181706800da628180671ab1219068054db0c1a06719b001b06815b80a11c1c068050b9301d066fac2a1e068033ab481f06719b1020068035ab721e07803dc2761f0771ae3c20078040f60e210771ce282207800ea4322307882a81a66024078035ad4625076efe7e26078162808e1827078118bb7228076eac7428088010bf58290871a04c2a0871bc722b086fa8382c08803d80a0142d088035d6282e088051c30c2f086efc623008800d9f6231086f986432088117bb7237028010a63034068010c84e2740800ea64c2237832c80933e1f3b830880c454390208813c80955c3905068032c73611348010a03c093c837a808a101b278050ac34093a8051ac34291b8f3b8187401d28881a82cb3a3a0a37977b86d20843000028996686a7083f030f8078d3761b27106e995a08499070839b5a1131000b00"_hex_u8;
/* 2023-11-07T17:59:35Z 48, 4792, 498995,*/
static constexpr auto BENCH_EXAMPLE_10 = "875f89aa1000b51ec09d7201c55cc7a72e02a11aa1fb3203b233a7f95204800ef56205b33ea9d13006803e80b26e07d90ec9dd4008b45eabbe6c09806080ca000a815984e8680a0a6f80925e0a0a803f80e1660c09937c94b7420d086e82f5640a086e80997e0b086f808d320c08800580a5640d086f8089100e08804080c9060f088115819a1c10086e82961a0f0a805f81bc0a100a6ff826110a6ef53e120a807584c60c110a6e818f32120a803c81c246130a805481d508140a8159838410150a7180a55c160a6f80821c170a6fe6101c066fe6101d06805080f854190a6e81b27c1a0a8155819c701e06805180ae0c21046e8b9a222501805180f53422001680f26880f8a62a220116803580da582007058153838e6e21000c800d80a712033a807681ae1c23000308834a82d36023020205815981e03a051a08001700"_hex_u8;
/* 2023-11-16T10:47:08Z 77, 473962, 486863,*/
static constexpr auto BENCH_EXAMPLE_11 = "801980c06000801980c06001801980c06002801980c06003801980c06004801980c06005801980c06006801980c06007801980c06008801980c06009801980c0600a801980c0600b801980c0600c801980c0600d801980c0600e801980c0600f801980c060108019d12c11800f80b1601111800f80b1601111801080b1601111800f80b160100e800f80b160100f801980c060110f800f80b160140d801180b1601111801180b160100d801180b160120c801180b1600f10801180b1600f11801980c0601011800f80b160140e800f80b160110f801980c060170a801180b1601210801980c060140f800f80b1601311801980c0602005801180b1601f07800f80b1601b0c800fca7c1611812081f9601638812081f9601637812081fb001636801080b160142f801980c0600e2a801080b1600f2a801180b1600d25801980c0600e25800f80b1600d27801980c0600e27801980c0600d27801180b1600e26812080b1500c27812081f960201025812081f960200f27812081fc201d101c812081fc201d101d812081fc201d0f1f812081fc201d0f20812081f9601b1016800f80b1600a35800f80b1600a36800f80b1600e32801080b160122f812081f960280040812081fc20121d1b812081f960112713812081f960160d37812081fc20140d2b812081f960130d2d812081fc20130c2c812081fb001b0157812081fb001a0245812081fc20140030812081fc20092747812081fb000b152500"_hex_u8;
/* 2023-10-06T20:44:09Z 40, 341438, 341438,*/
static constexpr auto BENCH_EXAMPLE_12 = "80318f4c0080318f4c0180318f4c0280318f4c0380318f4c0480318f4c0580318f4c0680318f4c078033a57807078033a57807078033a57807078033a57807078033a57807078033a57807078033a57807078033a578070780318f4c0e0180318f4c0d0380318f4c0c0580318f4c0b078033a57803128033a57803128033a57803128033a578031280318f4c0412810b9c28140300810c9c281303028033a57802188033a57802188033a5780218810c9c280b01108033a578001c810c9c2807050f8033a578001b810c98040700158033a578001c810c98040301158033a5780019806ca1240101118033a578001300"_hex_u8;
/* 2023-11-15T21:40:46Z 96, 23608, 138286,*/
static constexpr auto BENCH_EXAMPLE_13 = "8060829f4000b157bab07a01b27cc2b16802b22fbce54603826480a95804803da81a05bc7bcac93806800de55207800daf0608805bc71809805bc7180a800d9d4a0b805bbc700c8152d7180d805bb9380e850a8886260f800d80d33410bf38d3d55011b41dc4eb6012bd70d2ce2e138d3596af7812137180cd501313805e81f7281413718092001513803d81f90016136e8b916c1713801081861a17106e80cd2a18106f80cc3c19106e80cf161911800d80fe781b107180d87c1c106e80fb081d10803e8286701d11800d81c4781f10804082a6002010801081912e21107180ff0021116e81da4a2310850b8b864023116e89db3224116e84ff7e2610897c95993427106f80bb1a240b803581c272250b8032828c10260b6e80d42a270b804082b35a280b800d80fe3e290b805cc0282312821d8697022b0b6e8add562c0b805281c8063007811883f1082313800d80fe3e24137180c9142513800d8380102613803382c00e2713805eb32228136e8494542913800e8186742913806082b74c2a1380528285782b13800d818f7a2c136e84a5562d1380508286702e136f80a46e3e04803f8191364102805481ad4c3d076e809a5a3e077180fe4032136e838b7233138c4790cf384106853584ab624206805b80932a4801806280966c48028168ef04400b7181bd524903806282db5c375b9316acbf703a599c68c5a454385c6e81d63e364a6f80ff64334e817485a6784f023171819536234e800d81826e1e498053829a12420018834c87cb14291d2e840e8bc94c1d2825800d81b7220368811783fe0e271f1f811783e758380f001ecd55809edf6e56000000003a815984ba76008010d54d80aebb4e2c22000000000000002c807682f150007a00"_hex_u8;
/* 2023-12-06T09:18:20Z 93, 68130, 122830,*/
static constexpr auto BENCH_EXAMPLE_14 = "b26beadb2e00800d80ca0a01d41c80b1e14c02b068e8883003800d81af1604b34beede30056e80b14006b151f5d46c07b93e8085b02608b30cf98b1009b14ef6b3040ab176f6ab480bb7078082b8640c800d81c6460d802c80a8080e802c80a8080f802c80a14210802ce50a11802cd722127181ce6012126e81d14a13126e9b8b00141282428dd42c15128051828408150e6e81bd3e150f805f84ad00160f7181de30170f6e81c740180f800d83b876190f6e82a5541a0f6e81d33e1a106e82a70a1b106f81e76e1c10803583f2081d106e82d9401e106e96e4441f107181de321e12815889e2341f127182d60c20126e979d4e21126e8282262410800d82972c25106f838a5822126f82842a23127182d24a2412803e84bc2a2512800d83c81a26126e84f8142712805085a22c27126e889e6a2812801083aa50281280348598102912801082d5522a126e85865c2b127182c7602b1282468c82042c126e84972c2d12805485d93a2d12801083c7322e12815386e1582f126e84fb0c30126f82eb6c3011813a85b47a3111803f869f5c3211805181ed30370d6e84bf0a3411804180e1383809815883aa183a08815a8392203e05807681f140380c6e9e4c4005805485ab363255805183856030406e82f9582c45805185c1001b4f82418df1001a4e803283c50e430026800d83a6201a4b836886be3044010b8b318988084c0101803183a6120776800d828a1e087682338ae050301c33873199f8624d010032813986bc663c1034800d83a5220a6f800d82be52048000805183e364084907800d83cc4a018005815987b41e1832000017884b9dce72035035803284c11e00800885769d9538192f0000000002001000"_hex_u8;
static void LinearizeOptimallyExample00(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_00); }
static void LinearizeOptimallyExample01(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_01); }
static void LinearizeOptimallyExample02(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_02); }
@ -328,6 +335,10 @@ static void LinearizeOptimallyExample07(benchmark::Bench& bench) { BenchLineariz
static void LinearizeOptimallyExample08(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_08); }
static void LinearizeOptimallyExample09(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_09); }
static void LinearizeOptimallyExample10(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_10); }
static void LinearizeOptimallyExample11(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_11); }
static void LinearizeOptimallyExample12(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_12); }
static void LinearizeOptimallyExample13(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_13); }
static void LinearizeOptimallyExample14(benchmark::Bench& bench) { BenchLinearizeOptimally(bench, BENCH_EXAMPLE_14); }
BENCHMARK(Linearize16TxWorstCase20Iters, benchmark::PriorityLevel::HIGH);
BENCHMARK(Linearize16TxWorstCase120Iters, benchmark::PriorityLevel::HIGH);
@ -381,3 +392,7 @@ BENCHMARK(LinearizeOptimallyExample07, benchmark::PriorityLevel::HIGH);
BENCHMARK(LinearizeOptimallyExample08, benchmark::PriorityLevel::HIGH);
BENCHMARK(LinearizeOptimallyExample09, benchmark::PriorityLevel::HIGH);
BENCHMARK(LinearizeOptimallyExample10, benchmark::PriorityLevel::HIGH);
BENCHMARK(LinearizeOptimallyExample11, benchmark::PriorityLevel::HIGH);
BENCHMARK(LinearizeOptimallyExample12, benchmark::PriorityLevel::HIGH);
BENCHMARK(LinearizeOptimallyExample13, benchmark::PriorityLevel::HIGH);
BENCHMARK(LinearizeOptimallyExample14, benchmark::PriorityLevel::HIGH);

29
src/cluster_linearize.h
View File

@ -647,7 +647,7 @@ public:
* be <= max_iterations. If strictly < max_iterations, the
* returned subset is optimal.
*
* Complexity: O(N * min(max_iterations, 2^N)) where N=depgraph.TxCount().
* Complexity: possibly O(N * min(max_iterations, sqrt(2^N))) where N=depgraph.TxCount().
*/
std::pair<SetInfo<SetType>, uint64_t> FindCandidateSet(uint64_t max_iterations, SetInfo<SetType> best) noexcept
{
@ -723,7 +723,8 @@ public:
}
/** Internal function to add an item to the queue of elements to explore if there are any
* transactions left to split on, and to update best/imp.
* transactions left to split on, possibly improving it before doing so, and to update
* best/imp.
*
* - inc: the "inc" value for the new work item (must be topological).
* - und: the "und" value for the new work item ((inc | und) must be topological).
@ -746,6 +747,28 @@ public:
pot.Set(m_sorted_depgraph, pos);
}
// The "jump ahead" optimization: whenever pot has a topologically-valid subset,
// that subset can be added to inc. Any subset of (pot - inc) has the property that
// its feerate exceeds that of any set compatible with this work item (superset of
// inc, subset of (inc | und)). Thus, if T is a topological subset of pot, and B is
// the best topologically-valid set compatible with this work item, and (T - B) is
// non-empty, then (T | B) is better than B and also topological. This is in
// contradiction with the assumption that B is best. Thus, (T - B) must be empty,
// or T must be a subset of B.
//
// See https://delvingbitcoin.org/t/how-to-linearize-your-cluster/303 section 2.4.
const auto init_inc = inc.transactions;
for (auto pos : pot.transactions - inc.transactions) {
// If the transaction's ancestors are a subset of pot, we can add it together
// with its ancestors to inc. Just update the transactions here; the feerate
// update happens below.
auto anc_todo = m_sorted_depgraph.Ancestors(pos) & m_todo;
if (anc_todo.IsSubsetOf(pot.transactions)) inc.transactions |= anc_todo;
}
// Finally update und and inc's feerate to account for the added transactions.
und -= inc.transactions;
inc.feerate += m_sorted_depgraph.FeeRate(inc.transactions - init_inc);
// If inc's feerate is better than best's, remember it as our new best.
if (inc.feerate > best.feerate) {
best = inc;
@ -892,7 +915,7 @@ public:
* - A boolean indicating whether the result is guaranteed to be
* optimal.
*
* Complexity: O(N * min(max_iterations + N, 2^N)) where N=depgraph.TxCount().
* Complexity: possibly O(N * min(max_iterations + N, sqrt(2^N))) where N=depgraph.TxCount().
*/
template<typename SetType>
std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& depgraph, uint64_t max_iterations, uint64_t rng_seed, Span<const ClusterIndex> old_linearization = {}) noexcept

13
src/test/fuzz/cluster_linearize.cpp
View File

@ -553,6 +553,12 @@ FUZZ_TARGET(clusterlin_search_finder)
// longer connected after removing certain transactions, this holds, because the connected
// components are searched separately.
assert(iterations_done <= (uint64_t{1} << (todo.Count() - 1)));
// Additionally, test that no more than sqrt(2^N)+1 iterations are required. This is just
// an empirical bound that seems to hold, without proof. Still, add a test for it so we
// can learn about counterexamples if they exist.
if (iterations_done >= 1 && todo.Count() <= 63) {
Assume((iterations_done - 1) * (iterations_done - 1) <= uint64_t{1} << todo.Count());
}
// Perform quality checks only if SearchCandidateFinder claims an optimal result.
if (iterations_done < max_iterations) {
@ -769,6 +775,13 @@ FUZZ_TARGET(clusterlin_linearize)
if (n <= 19 && iter_count > (uint64_t{1} << n)) {
assert(optimal);
}
// Additionally, if the assumption of sqrt(2^k)+1 iterations per step holds, the maximum number
// of iterations is also bounded by (2 + sqrt(2)) * (sqrt(2^n) - 1) + n, which is less than
// (2 + sqrt(2)) * sqrt(2^n) + n. Subtracting n and squaring gives
// (6 + 4 * sqrt(2)) * 2^n < 12 * 2^n.
if (n <= 35 && iter_count > n && (iter_count - n) * (iter_count - n) >= uint64_t{12} << n) {
Assume(optimal);
}
// If Linearize claims optimal result, run quality tests.
if (optimal) {