From 63dd9fd09e81e74c2369891762d5da84191fe197 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 9 Mar 2023 17:45:08 +0900 Subject: [PATCH 1/6] Log a warn if there are lot of 404 from esplora tx api --- backend/src/api/mempool.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4c475502c..db5de82b2 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -31,6 +31,11 @@ class Mempool { private mempoolProtection = 0; private latestTransactions: any[] = []; + private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100; + private SAMPLE_TIME = 10000; // In ms + private timer = new Date().getTime(); + private missingTxCount = 0; + constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); setInterval(this.deleteExpiredTransactions.bind(this), 20000); @@ -128,6 +133,16 @@ class Mempool { loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100); } + // https://github.com/mempool/mempool/issues/3283 + const logEsplora404 = (missingTxCount, threshold, time) => { + const log = `In the past ${time / 1000} seconds, esplora tx API replied ${missingTxCount} times with a 404 error code while updating nodejs backend mempool`; + if (missingTxCount >= threshold) { + logger.warn(log); + } else if (missingTxCount > 0) { + logger.debug(log); + } + }; + for (const txid of transactions) { if (!this.mempoolCache[txid]) { try { @@ -142,7 +157,10 @@ class Mempool { } hasChange = true; newTransactions.push(transaction); - } catch (e) { + } catch (e: any) { + if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { + this.missingTxCount++; + } logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); } } @@ -152,6 +170,14 @@ class Mempool { } } + // Reset esplora 404 counter and log a warning if needed + const elapsedTime = new Date().getTime() - this.timer; + if (elapsedTime > this.SAMPLE_TIME) { + logEsplora404(this.missingTxCount, this.ESPLORA_MISSING_TX_WARNING_THRESHOLD, elapsedTime); + this.timer = new Date().getTime(); + this.missingTxCount = 0; + } + // Prevent mempool from clear on bitcoind restart by delaying the deletion if (this.mempoolProtection === 0 && currentMempoolSize > 20000 From 6327ce7c899878411d4777151bc0bba425134bfb Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 10 Mar 2023 09:21:44 +0900 Subject: [PATCH 2/6] Sort asn numerically - add few more top 10 isp in warm cache --- backend/src/api/explorer/nodes.api.ts | 16 ++++++++-------- production/nginx-cache-warmer | 7 +++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index e1949a17c..d86ecf665 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -417,24 +417,24 @@ class NodesApi { if (!ispList[isp1]) { ispList[isp1] = { - id: channel.isp1ID.toString(), + ids: [channel.isp1ID], capacity: 0, channels: 0, nodes: {}, }; - } else if (ispList[isp1].id.indexOf(channel.isp1ID) === -1) { - ispList[isp1].id += ',' + channel.isp1ID.toString(); + } else if (ispList[isp1].ids.includes(channel.isp1ID) === false) { + ispList[isp1].ids.push(channel.isp1ID); } if (!ispList[isp2]) { ispList[isp2] = { - id: channel.isp2ID.toString(), + ids: [channel.isp2ID], capacity: 0, channels: 0, nodes: {}, }; - } else if (ispList[isp2].id.indexOf(channel.isp2ID) === -1) { - ispList[isp2].id += ',' + channel.isp2ID.toString(); + } else if (ispList[isp2].ids.includes(channel.isp2ID) === false) { + ispList[isp2].ids.push(channel.isp2ID); } ispList[isp1].capacity += channel.capacity; @@ -444,11 +444,11 @@ class NodesApi { ispList[isp2].channels += 1; ispList[isp2].nodes[channel.node2PublicKey] = true; } - + const ispRanking: any[] = []; for (const isp of Object.keys(ispList)) { ispRanking.push([ - ispList[isp].id, + ispList[isp].ids.sort((a, b) => a - b).join(','), isp, ispList[isp].capacity, ispList[isp].channels, diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index 5a298367b..6b58e5b78 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -95,7 +95,7 @@ do for url in / \ '/api/v1/lightning/statistics/3y' \ '/api/v1/lightning/statistics/all' \ '/api/v1/lightning/nodes/isp-ranking' \ - '/api/v1/lightning/nodes/isp/396982,15169' `# Google` \ + '/api/v1/lightning/nodes/isp/15169,396982' `# Google` \ '/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` \ '/api/v1/lightning/nodes/isp/39572' `# DataWeb` \ '/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \ @@ -107,12 +107,15 @@ do for url in / \ '/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` \ '/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` \ '/api/v1/lightning/nodes/isp/16276' `# OVH SAS` \ - '/api/v1/lightning/nodes/isp/11426,11427,20001,20115,11351,10796,33363,12271' `# Spectrum` \ + '/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,20115,33363' `# Spectrum` \ '/api/v1/lightning/nodes/isp/701' `# Verizon` \ '/api/v1/lightning/nodes/isp/12876' `# Scaleway` \ '/api/v1/lightning/nodes/isp/33915' `# Ziggo` \ '/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` \ '/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` \ + '/api/v1/lightning/nodes/isp/212531', `# UAB Interneto vizija` \ + '/api/v1/lightning/nodes/isp/63949', `# Linode` \ + '/api/v1/lightning/nodes/isp/51167', `# Contabo GmbH` \ '/api/v1/lightning/nodes/countries' \ '/api/v1/lightning/nodes/rankings' \ '/api/v1/lightning/nodes/rankings/liquidity' \ From a2d673f9ede75de63026a322c4f9dc3b537b4a7c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 10 Mar 2023 00:27:22 -0600 Subject: [PATCH 3/6] batch address outspend lookups into <50 txids per request --- .../transactions-list.component.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index afda646d7..f499300c1 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit, Input, ChangeDetectionStrategy, OnChanges, Output, EventEmitter, ChangeDetectorRef } from '@angular/core'; import { StateService } from '../../services/state.service'; import { CacheService } from '../../services/cache.service'; -import { Observable, ReplaySubject, BehaviorSubject, merge, Subscription, of } from 'rxjs'; +import { Observable, ReplaySubject, BehaviorSubject, merge, Subscription, of, forkJoin } from 'rxjs'; import { Outspend, Transaction, Vin, Vout } from '../../interfaces/electrs.interface'; import { ElectrsApiService } from '../../services/electrs-api.service'; import { environment } from '../../../environments/environment'; @@ -70,12 +70,19 @@ export class TransactionsListComponent implements OnInit, OnChanges { .pipe( switchMap((txIds) => { if (!this.cached) { - return this.apiService.getOutspendsBatched$(txIds); + // break list into batches of 50 (maximum supported by esplora) + const batches = []; + for (let i = 0; i < txIds.length; i += 50) { + batches.push(txIds.slice(i, i + 50)); + } + return forkJoin(batches.map(batch => this.apiService.getOutspendsBatched$(batch))); } else { return of([]); } }), - tap((outspends: Outspend[][]) => { + tap((batchedOutspends: Outspend[][][]) => { + // flatten batched results back into a single array + const outspends = batchedOutspends.flat(1); if (!this.transactions) { return; } From ef0cc9d2db0e2b2253dc040890ad7c4a52c8c42f Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 11 Mar 2023 18:04:06 +0900 Subject: [PATCH 4/6] Changing interval to block time --- .../difficulty/difficulty.component.html | 2 +- frontend/src/locale/messages.xlf | 342 +++++++++++------- 2 files changed, 222 insertions(+), 122 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty.component.html b/frontend/src/app/components/difficulty/difficulty.component.html index 5b2f23cf3..d23edcfe3 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.html +++ b/frontend/src/app/components/difficulty/difficulty.component.html @@ -39,7 +39,7 @@
~
-
Average interval
+
Average block time
diff --git a/frontend/src/locale/messages.xlf b/frontend/src/locale/messages.xlf index 48c7da18d..e040e7bc9 100644 --- a/frontend/src/locale/messages.xlf +++ b/frontend/src/locale/messages.xlf @@ -2452,6 +2452,10 @@ src/app/components/block/block.component.html 8,9 + + src/app/components/difficulty/difficulty-tooltip.component.html + 38 + src/app/components/mempool-block/mempool-block.component.ts 75 @@ -2870,6 +2874,10 @@ Difficulty Adjustment + + src/app/components/difficulty-mining/difficulty-mining.component.html + 1,5 + src/app/components/difficulty/difficulty.component.html 1,5 @@ -2883,11 +2891,11 @@ Remaining - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 7,9 - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 66,69 difficulty-box.remaining @@ -2896,11 +2904,11 @@ blocks - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 10,11 - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 53,54 @@ -2921,11 +2929,11 @@ block - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 11,12 - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 54,55 @@ -2937,11 +2945,11 @@ Estimate - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 16,17 - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 73,76 difficulty-box.estimate @@ -2949,19 +2957,23 @@ Previous - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 31,33 + + src/app/components/difficulty/difficulty.component.html + 59,61 + difficulty-box.previous Current Period - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 43,44 - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 80,83 difficulty-box.current-period @@ -2969,11 +2981,99 @@ Next Halving - src/app/components/difficulty/difficulty.component.html + src/app/components/difficulty-mining/difficulty-mining.component.html 50,52 difficulty-box.next-halving + + blocks expected + + src/app/components/difficulty/difficulty-tooltip.component.html + 13 + + difficulty-box.expected-blocks + + + block expected + + src/app/components/difficulty/difficulty-tooltip.component.html + 14 + + difficulty-box.expected-block + + + blocks mined + + src/app/components/difficulty/difficulty-tooltip.component.html + 18 + + difficulty-box.mined-blocks + + + block mined + + src/app/components/difficulty/difficulty-tooltip.component.html + 19 + + difficulty-box.mined-block + + + blocks remaining + + src/app/components/difficulty/difficulty-tooltip.component.html + 24 + + difficulty-box.remaining-blocks + + + block remaining + + src/app/components/difficulty/difficulty-tooltip.component.html + 25 + + difficulty-box.remaining-block + + + blocks ahead + + src/app/components/difficulty/difficulty-tooltip.component.html + 29 + + difficulty-box.blocks-ahead + + + block ahead + + src/app/components/difficulty/difficulty-tooltip.component.html + 30 + + difficulty-box.block-ahead + + + blocks behind + + src/app/components/difficulty/difficulty-tooltip.component.html + 34 + + difficulty-box.blocks-behind + + + block behind + + src/app/components/difficulty/difficulty-tooltip.component.html + 35 + + difficulty-box.block-behind + + + Average block time + + src/app/components/difficulty/difficulty.component.html + 42,45 + + difficulty-box.average-block-time + Either 2x the minimum, or the Low Priority rate (whichever is lower) @@ -4054,39 +4154,27 @@ Just now src/app/components/time/time.component.ts - 78 + 79 ago - - src/app/components/time/time.component.ts - 97 - - - src/app/components/time/time.component.ts - 98 - - - src/app/components/time/time.component.ts - 99 - - - src/app/components/time/time.component.ts - 100 - - - src/app/components/time/time.component.ts - 101 - - - src/app/components/time/time.component.ts - 102 - src/app/components/time/time.component.ts 103 + + src/app/components/time/time.component.ts + 104 + + + src/app/components/time/time.component.ts + 105 + + + src/app/components/time/time.component.ts + 106 + src/app/components/time/time.component.ts 107 @@ -4099,53 +4187,53 @@ src/app/components/time/time.component.ts 109 - - src/app/components/time/time.component.ts - 110 - - - src/app/components/time/time.component.ts - 111 - - - src/app/components/time/time.component.ts - 112 - src/app/components/time/time.component.ts 113 + + src/app/components/time/time.component.ts + 114 + + + src/app/components/time/time.component.ts + 115 + + + src/app/components/time/time.component.ts + 116 + + + src/app/components/time/time.component.ts + 117 + + + src/app/components/time/time.component.ts + 118 + + + src/app/components/time/time.component.ts + 119 + In ~ - - src/app/components/time/time.component.ts - 120 - - - src/app/components/time/time.component.ts - 121 - - - src/app/components/time/time.component.ts - 122 - - - src/app/components/time/time.component.ts - 123 - - - src/app/components/time/time.component.ts - 124 - - - src/app/components/time/time.component.ts - 125 - src/app/components/time/time.component.ts 126 + + src/app/components/time/time.component.ts + 127 + + + src/app/components/time/time.component.ts + 128 + + + src/app/components/time/time.component.ts + 129 + src/app/components/time/time.component.ts 130 @@ -4158,53 +4246,53 @@ src/app/components/time/time.component.ts 132 - - src/app/components/time/time.component.ts - 133 - - - src/app/components/time/time.component.ts - 134 - - - src/app/components/time/time.component.ts - 135 - src/app/components/time/time.component.ts 136 + + src/app/components/time/time.component.ts + 137 + + + src/app/components/time/time.component.ts + 138 + + + src/app/components/time/time.component.ts + 139 + + + src/app/components/time/time.component.ts + 140 + + + src/app/components/time/time.component.ts + 141 + + + src/app/components/time/time.component.ts + 142 + After - - src/app/components/time/time.component.ts - 143 - - - src/app/components/time/time.component.ts - 144 - - - src/app/components/time/time.component.ts - 145 - - - src/app/components/time/time.component.ts - 146 - - - src/app/components/time/time.component.ts - 147 - - - src/app/components/time/time.component.ts - 148 - src/app/components/time/time.component.ts 149 + + src/app/components/time/time.component.ts + 150 + + + src/app/components/time/time.component.ts + 151 + + + src/app/components/time/time.component.ts + 152 + src/app/components/time/time.component.ts 153 @@ -4217,22 +4305,34 @@ src/app/components/time/time.component.ts 155 - - src/app/components/time/time.component.ts - 156 - - - src/app/components/time/time.component.ts - 157 - - - src/app/components/time/time.component.ts - 158 - src/app/components/time/time.component.ts 159 + + src/app/components/time/time.component.ts + 160 + + + src/app/components/time/time.component.ts + 161 + + + src/app/components/time/time.component.ts + 162 + + + src/app/components/time/time.component.ts + 163 + + + src/app/components/time/time.component.ts + 164 + + + src/app/components/time/time.component.ts + 165 + This transaction has been replaced by: From 23ea5d582b4fe200eaf115fa65eb1e2d6947568e Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 11 Mar 2023 18:32:47 +0900 Subject: [PATCH 5/6] Fixes i18n duplicate warning --- .../components/difficulty/difficulty-tooltip.component.html | 2 +- .../components/difficulty/difficulty-tooltip.component.scss | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html index 57a96cca7..d06bb5e91 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html @@ -35,7 +35,7 @@ {{ i }} block behind - next block + Next Block
\ No newline at end of file diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss b/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss index 82b762acd..8b7a5eb73 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss @@ -15,4 +15,8 @@ margin: 0; white-space: nowrap; } -} \ No newline at end of file +} + +.next-block { + text-transform: capitalize; +} From 6b4650f3cd24deab416dcb3113f70cbf51286601 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 11 Mar 2023 18:34:51 +0900 Subject: [PATCH 6/6] next block lower case css fix --- .../app/components/difficulty/difficulty-tooltip.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss b/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss index 8b7a5eb73..5b4a8a02f 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.scss @@ -18,5 +18,5 @@ } .next-block { - text-transform: capitalize; + text-transform: lowercase; }