mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-11-19 01:41:01 +01:00
Code Issues Projects Releases Wiki Activity

Configurable esplora batch sizes

Browse Source
This commit is contained in:
Mononaut 2023-11-15 06:10:00 +00:00
parent 1aea3fcac5
commit 35d7c55c1d
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E
11 changed files with 30 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/mempool-config.sample.json
View File

@ -52,6 +52,7 @@
"ESPLORA": {
"REST_API_URL": "http://127.0.0.1:3000",
"UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
"BATCH_QUERY_BASE_SIZE": 1000,
"RETRY_UNIX_SOCKET_AFTER": 30000,
"REQUEST_TIMEOUT": 10000,
"FALLBACK_TIMEOUT": 5000,
@ -132,6 +133,11 @@
"BISQ_URL": "https://bisq.markets/api",
"BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
},
"REDIS": {
"ENABLED": false,
"UNIX_SOCKET_PATH": "/tmp/redis.sock",
"BATCH_QUERY_BASE_SIZE": 5000
},
"REPLICATION": {
"ENABLED": false,
"AUDIT": false,

4
backend/src/__fixtures__/mempool-config.template.json
View File

@ -53,6 +53,7 @@
"ESPLORA": {
"REST_API_URL": "__ESPLORA_REST_API_URL__",
"UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
"BATCH_QUERY_BASE_SIZE": 1000,
"RETRY_UNIX_SOCKET_AFTER": 888,
"REQUEST_TIMEOUT": 10000,
"FALLBACK_TIMEOUT": 5000,
@ -140,6 +141,7 @@
},
"REDIS": {
"ENABLED": false,
"UNIX_SOCKET_PATH": "/tmp/redis.sock"
"UNIX_SOCKET_PATH": "/tmp/redis.sock",
"BATCH_QUERY_BASE_SIZE": 5000
}
}

4
backend/src/__tests__/config.test.ts
View File

@ -55,6 +55,7 @@ describe('Mempool Backend Config', () => {
expect(config.ESPLORA).toStrictEqual({
REST_API_URL: 'http://127.0.0.1:3000',
UNIX_SOCKET_PATH: null,
BATCH_QUERY_BASE_SIZE: 1000,
RETRY_UNIX_SOCKET_AFTER: 30000,
REQUEST_TIMEOUT: 10000,
FALLBACK_TIMEOUT: 5000,
@ -144,7 +145,8 @@ describe('Mempool Backend Config', () => {
expect(config.REDIS).toStrictEqual({
ENABLED: false,
UNIX_SOCKET_PATH: ''
UNIX_SOCKET_PATH: '',
BATCH_QUERY_BASE_SIZE: 5000,
});
});
});

2
backend/src/api/mempool.ts
View File

@ -235,7 +235,7 @@ class Mempool {
if (!loaded) {
const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]);
const sliceLength = 10000;
const sliceLength = config.ESPLORA.BATCH_QUERY_BASE_SIZE;
for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) {
const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength);
const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false);

2
backend/src/api/rbf-cache.ts
View File

@ -480,7 +480,7 @@ class RbfCache {
};
if (config.MEMPOOL.BACKEND === 'esplora') {
const sliceLength = 250;
const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 40);
for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) {
const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength);
try {

5
backend/src/api/redis-cache.ts
View File

@ -122,8 +122,9 @@ class RedisCache {
async $removeTransactions(transactions: string[]) {
try {
await this.$ensureConnected();
for (let i = 0; i < Math.ceil(transactions.length / 10000); i++) {
const slice = transactions.slice(i * 10000, (i + 1) * 10000);
const sliceLength = config.REDIS.BATCH_QUERY_BASE_SIZE;
for (let i = 0; i < Math.ceil(transactions.length / sliceLength); i++) {
const slice = transactions.slice(i * sliceLength, (i + 1) * sliceLength);
await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`));
logger.debug(`Deleted ${slice.length} transactions from the Redis cache`);
}

4
backend/src/config.ts
View File

@ -43,6 +43,7 @@ interface IConfig {
ESPLORA: {
REST_API_URL: string;
UNIX_SOCKET_PATH: string | void | null;
BATCH_QUERY_BASE_SIZE: number;
RETRY_UNIX_SOCKET_AFTER: number;
REQUEST_TIMEOUT: number;
FALLBACK_TIMEOUT: number;
@ -151,6 +152,7 @@ interface IConfig {
REDIS: {
ENABLED: boolean;
UNIX_SOCKET_PATH: string;
BATCH_QUERY_BASE_SIZE: number;
},
}
@ -195,6 +197,7 @@ const defaults: IConfig = {
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',
'UNIX_SOCKET_PATH': null,
'BATCH_QUERY_BASE_SIZE': 1000,
'RETRY_UNIX_SOCKET_AFTER': 30000,
'REQUEST_TIMEOUT': 10000,
'FALLBACK_TIMEOUT': 5000,
@ -303,6 +306,7 @@ const defaults: IConfig = {
'REDIS': {
'ENABLED': false,
'UNIX_SOCKET_PATH': '',
'BATCH_QUERY_BASE_SIZE': 5000,
},
};

2
backend/src/tasks/lightning/forensics.service.ts
View File

@ -79,7 +79,7 @@ class ForensicsService {
}
let progress = 0;
const sliceLength = 1000;
const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 10);
// process batches of 1000 channels
for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) {
const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength);

2
backend/src/tasks/lightning/network-sync.service.ts
View File

@ -290,7 +290,7 @@ class NetworkSyncService {
const allChannels = await channelsApi.$getChannelsByStatus([0, 1]);
const sliceLength = 5000;
const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 2);
// process batches of 5000 channels
for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) {
const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength);

4
docker/backend/mempool-config.json
View File

@ -53,6 +53,7 @@
"ESPLORA": {
"REST_API_URL": "__ESPLORA_REST_API_URL__",
"UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
"BATCH_QUERY_BASE_SIZE": __ESPLORA_BATCH_QUERY_BASE_SIZE__,
"RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__,
"REQUEST_TIMEOUT": __ESPLORA_REQUEST_TIMEOUT__,
"FALLBACK_TIMEOUT": __ESPLORA_FALLBACK_TIMEOUT__,
@ -146,6 +147,7 @@
},
"REDIS": {
"ENABLED": __REDIS_ENABLED__,
"UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__"
"UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__",
"BATCH_QUERY_BASE_SIZE": __REDIS_BATCH_QUERY_BASE_SIZE__
}
}

4
docker/backend/start.sh
View File

@ -54,6 +54,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
# ESPLORA
__ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"}
__ESPLORA_BATCH_QUERY_BASE_SIZE__=${ESPLORA_BATCH_QUERY_BASE_SIZE:=1000}
__ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000}
__ESPLORA_REQUEST_TIMEOUT__=${ESPLORA_REQUEST_TIMEOUT:=5000}
__ESPLORA_FALLBACK_TIMEOUT__=${ESPLORA_FALLBACK_TIMEOUT:=5000}
@ -148,6 +149,7 @@ __MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false}
# REDIS
__REDIS_ENABLED__=${REDIS_ENABLED:=false}
__REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=true}
__REDIS_BATCH_QUERY_BASE_SIZE__=${REDIS_BATCH_QUERY_BASE_SIZE:=5000}
mkdir -p "${__MEMPOOL_CACHE_DIR__}"
@ -201,6 +203,7 @@ sed -i "s!__ELECTRUM_TLS_ENABLED__!${__ELECTRUM_TLS_ENABLED__}!g" mempool-config
sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json
sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json
sed -i "s!__ESPLORA_BATCH_QUERY_BASE_SIZE__!${__ESPLORA_BATCH_QUERY_BASE_SIZE__}!g" mempool-config.json
sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json
sed -i "s!__ESPLORA_REQUEST_TIMEOUT__!${__ESPLORA_REQUEST_TIMEOUT__}!g" mempool-config.json
sed -i "s!__ESPLORA_FALLBACK_TIMEOUT__!${__ESPLORA_FALLBACK_TIMEOUT__}!g" mempool-config.json
@ -288,5 +291,6 @@ sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS_
# REDIS
sed -i "s!__REDIS_ENABLED__!${__REDIS_ENABLED__}!g" mempool-config.json
sed -i "s!__REDIS_UNIX_SOCKET_PATH__!${__REDIS_UNIX_SOCKET_PATH__}!g" mempool-config.json
sed -i "s!__REDIS_BATCH_QUERY_BASE_SIZE__!${__REDIS_BATCH_QUERY_BASE_SIZE__}!g" mempool-config.json
node /backend/package/index.js