mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-11-20 02:11:49 +01:00
Code Issues Projects Releases Wiki Activity

handle gaps in indexed CPFP data

Browse Source
This commit is contained in:
Mononaut 2023-01-10 17:15:55 -06:00
parent 8de3fd0988
commit f0d3bb87c6
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E
3 changed files with 35 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
backend/src/api/blocks.ts
View File

@ -338,10 +338,10 @@ class Blocks {
try {
// Get all indexed block hash
const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks();
logger.info(`Indexing cpfp data for ${unindexedBlocks.length} blocks`);
const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
if (!unindexedBlocks?.length) {
if (!unindexedBlockHeights?.length) {
return;
}
@ -350,32 +350,26 @@ class Blocks {
let countThisRun = 0;
let timer = new Date().getTime() / 1000;
const startedAt = new Date().getTime() / 1000;
let lastHeight;
for (const block of unindexedBlocks) {
for (const height of unindexedBlockHeights) {
// Logging
const hash = await bitcoinApi.$getBlockHash(height);
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = (countThisRun / elapsedSeconds);
const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`);
timer = new Date().getTime() / 1000;
countThisRun = 0;
}
await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block
await this.$indexCPFP(hash, height); // Calculate and save CPFP data for transactions in this block
lastHeight = block.height;
// Logging
count++;
countThisRun++;
}
if (count > 0) {
await cpfpRepository.$insertProgressMarker(lastHeight);
logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
} else {
logger.debug(`CPFP indexing completed: indexed ${count} blocks`);
}
logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
} catch (e) {
logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
throw e;
@ -790,7 +784,10 @@ class Blocks {
ancestors[vin.txid] = true;
});
}
await cpfpRepository.$batchSaveClusters(clusters);
const result = await cpfpRepository.$batchSaveClusters(clusters);
if (!result) {
await cpfpRepository.$insertProgressMarker(height);
}
}
}

24
backend/src/repositories/BlocksRepository.ts
View File

@ -671,19 +671,25 @@ class BlocksRepository {
try {
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
const currentBlockHeight = blockchainInfo.blocks;
const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from compact_cpfp_clusters`);
const lastHeight = (lastHeightRows.length && lastHeightRows[0].minHeight != null) ? lastHeightRows[0].minHeight : currentBlockHeight;
let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight);
if (indexingBlockAmount <= -1) {
indexingBlockAmount = currentBlockHeight + 1;
}
const firstHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
if (firstHeight < lastHeight) {
const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC`, [firstHeight, lastHeight]);
return rows;
}
const [rows]: any[] = await DB.query(`
SELECT height
FROM compact_cpfp_clusters
WHERE height <= ? AND height >= ?
ORDER BY height DESC;
`, [currentBlockHeight, minHeight]);
const indexedHeights = {};
rows.forEach((row) => { indexedHeights[row.height] = true; });
const allHeights: number[] = Array.from(Array(currentBlockHeight - minHeight + 1).keys(), n => n + minHeight).reverse();
const unindexedHeights = allHeights.filter(x => !indexedHeights[x]);
return unindexedHeights;
} catch (e) {
logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
throw e;

13
backend/src/repositories/CpfpRepository.ts
View File

@ -6,9 +6,9 @@ import { Ancestor } from '../mempool.interfaces';
import transactionRepository from '../repositories/TransactionRepository';
class CpfpRepository {
public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<boolean> {
if (!txs[0]) {
return;
return false;
}
// skip clusters of transactions with the same fees
const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
@ -16,7 +16,7 @@ class CpfpRepository {
return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
}, true);
if (equalFee) {
return;
return false;
}
try {
@ -41,13 +41,14 @@ class CpfpRepository {
await transactionRepository.$batchSetCluster(chunk);
chunkIndex += maxChunk;
}
return true;
} catch (e: any) {
logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<void> {
public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
try {
const clusterValues: any[] = [];
const txs: any[] = [];
@ -73,7 +74,7 @@ class CpfpRepository {
}
if (!clusterValues.length) {
return;
return false;
}
const maxChunk = 100;
@ -103,7 +104,7 @@ class CpfpRepository {
);
chunkIndex += maxChunk;
}
return;
return true;
} catch (e: any) {
logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;