mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-02-23 14:40:38 +01:00
Code Issues Projects Releases Wiki Activity

calculate & index cpfp packages

Browse source
This commit is contained in:
Mononaut 2022-11-27 13:46:23 +09:00
parent 3406758fd2
commit 9b6a012476
No known key found for this signature in database
GPG key ID: A3F058E41374C04E
13 changed files with 295 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
backend/mempool-config.sample.json
View file

@ -25,7 +25,8 @@
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"ADVANCED_TRANSACTION_SELECTION": false
"ADVANCED_TRANSACTION_SELECTION": false,
"TRANSACTION_INDEXING": false
},
"CORE_RPC": {
"HOST": "127.0.0.1",

3
backend/src/__fixtures__/mempool-config.template.json
View file

@ -26,7 +26,8 @@
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__"
"ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__",
"TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

1
backend/src/__tests__/config.test.ts
View file

@ -39,6 +39,7 @@ describe('Mempool Backend Config', () => {
POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
ADVANCED_TRANSACTION_SELECTION: false,
TRANSACTION_INDEXING: false,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

38
backend/src/api/bitcoin/bitcoin.routes.ts
View file

@ -17,13 +17,14 @@ import logger from '../../logger';
import blocks from '../blocks';
import bitcoinClient from './bitcoin-client';
import difficultyAdjustment from '../difficulty-adjustment';
import transactionRepository from '../../repositories/TransactionRepository';
class BitcoinRoutes {
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
.get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
.get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.getCpfpInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
.get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
@ -188,29 +189,34 @@ class BitcoinRoutes {
}
}
private getCpfpInfo(req: Request, res: Response) {
private async $getCpfpInfo(req: Request, res: Response) {
if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
res.status(501).send(`Invalid transaction ID.`);
return;
}
const tx = mempool.getMempool()[req.params.txId];
if (!tx) {
res.status(404).send(`Transaction doesn't exist in the mempool.`);
if (tx) {
if (tx?.cpfpChecked) {
res.json({
ancestors: tx.ancestors,
bestDescendant: tx.bestDescendant || null,
});
return;
}
const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
res.json(cpfpInfo);
return;
} else {
const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
if (cpfpInfo) {
res.json(cpfpInfo);
return;
}
}
if (tx.cpfpChecked) {
res.json({
ancestors: tx.ancestors,
bestDescendant: tx.bestDescendant || null,
});
return;
}
const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
res.json(cpfpInfo);
res.status(404).send(`Transaction has no CPFP info available.`);
}
private getBackendInfo(req: Request, res: Response) {

93
backend/src/api/blocks.ts
View file

@ -21,6 +21,8 @@ import fiatConversion from './fiat-conversion';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import cpfpRepository from '../repositories/CpfpRepository';
import transactionRepository from '../repositories/TransactionRepository';
import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import PricesRepository from '../repositories/PricesRepository';
@ -260,7 +262,7 @@ class Blocks {
/**
* [INDEXING] Index all blocks summaries for the block txs visualization
*/
public async $generateBlocksSummariesDatabase() {
public async $generateBlocksSummariesDatabase(): Promise<void> {
if (Common.blocksSummariesIndexingEnabled() === false) {
return;
}
@ -316,6 +318,56 @@ class Blocks {
}
}
/**
* [INDEXING] Index transaction CPFP data for all blocks
*/
public async $generateCPFPDatabase(): Promise<void> {
if (Common.cpfpIndexingEnabled() === false) {
return;
}
try {
// Get all indexed block hash
const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks();
if (!unindexedBlocks?.length) {
return;
}
// Logging
let count = 0;
let countThisRun = 0;
let timer = new Date().getTime() / 1000;
const startedAt = new Date().getTime() / 1000;
for (const block of unindexedBlocks) {
// Logging
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds);
const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
timer = new Date().getTime() / 1000;
countThisRun = 0;
}
await this.$indexCPFP(block.hash); // Calculate and save CPFP data for transactions in this block
// Logging
count++;
}
if (count > 0) {
logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
} else {
logger.debug(`CPFP indexing completed: indexed ${count} blocks`);
}
} catch (e) {
logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
throw e;
}
}
/**
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
@ -461,9 +513,13 @@ class Blocks {
await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await HashratesRepository.$deleteLastEntries();
await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock['height'] - i);
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
if (config.MEMPOOL.TRANSACTION_INDEXING) {
await this.$indexCPFP(newBlock.id);
}
}
await mining.$indexDifficultyAdjustments();
await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
@ -489,6 +545,9 @@ class Blocks {
if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true);
}
if (config.MEMPOOL.TRANSACTION_INDEXING) {
this.$indexCPFP(blockExtended.id);
}
}
}
@ -678,6 +737,38 @@ class Blocks {
public getCurrentBlockHeight(): number {
return this.currentBlockHeight;
}
public async $indexCPFP(hash: string): Promise<void> {
const block = await bitcoinClient.getBlock(hash, 2);
const transactions = block.tx;
let cluster: IBitcoinApi.VerboseTransaction[] = [];
let ancestors: { [txid: string]: boolean } = {};
for (let i = transactions.length - 1; i >= 0; i--) {
const tx = transactions[i];
if (!ancestors[tx.txid]) {
let totalFee = 0;
let totalWeight = 0;
cluster.forEach(tx => {
totalFee += tx?.fee || 0;
totalWeight += tx.weight;
});
const effectiveFeePerVsize = (totalFee * 100_000_000) / (totalWeight / 4);
if (cluster.length > 1) {
await cpfpRepository.$saveCluster(block.height, cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: (tx.fee || 0) * 100_000_000 }; }), effectiveFeePerVsize);
for (const tx of cluster) {
await transactionRepository.$setCluster(tx.txid, cluster[0].txid);
}
}
cluster = [];
ancestors = {};
}
cluster.push(tx);
tx.vin.forEach(vin => {
ancestors[vin.txid] = true;
});
}
await blocksRepository.$setCPFPIndexed(hash);
}
}
export default new Blocks();

7
backend/src/api/common.ts
View file

@ -187,6 +187,13 @@ export class Common {
);
}
static cpfpIndexingEnabled(): boolean {
return (
Common.indexingEnabled() &&
config.MEMPOOL.TRANSACTION_INDEXING === true
);
}
static setDateMidnight(date: Date): void {
date.setUTCHours(0);
date.setUTCMinutes(0);

27
backend/src/api/database-migration.ts
View file

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 45;
private static currentVersion = 46;
private queryTimeout = 900_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -369,6 +369,12 @@ class DatabaseMigration {
if (databaseSchemaVersion < 45 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
}
if (databaseSchemaVersion < 46 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD cpfp_indexed tinyint(1) DEFAULT 0');
await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
}
}
/**
@ -817,6 +823,25 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateCPFPTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS cpfp_clusters (
root varchar(65) NOT NULL,
height int(10) NOT NULL,
txs JSON DEFAULT NULL,
fee_rate double unsigned NOT NULL,
PRIMARY KEY (root)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateTransactionsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS transactions (
txid varchar(65) NOT NULL,
cluster varchar(65) DEFAULT NULL,
PRIMARY KEY (txid),
FOREIGN KEY (cluster) REFERENCES cpfp_clusters (root) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];

2
backend/src/config.ts
View file

@ -30,6 +30,7 @@ interface IConfig {
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
ADVANCED_TRANSACTION_SELECTION: boolean;
TRANSACTION_INDEXING: boolean;
};
ESPLORA: {
REST_API_URL: string;
@ -148,6 +149,7 @@ const defaults: IConfig = {
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'ADVANCED_TRANSACTION_SELECTION': false,
'TRANSACTION_INDEXING': false,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

1
backend/src/indexer.ts
View file

@ -77,6 +77,7 @@ class Indexer {
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();
await blocks.$generateCPFPDatabase();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

4
backend/src/mempool.interfaces.ts
View file

@ -119,7 +119,9 @@ interface BestDescendant {
export interface CpfpInfo {
ancestors: Ancestor[];
bestDescendant: BestDescendant | null;
bestDescendant?: BestDescendant | null;
descendants?: Ancestor[];
effectiveFeePerVsize?: number;
}
export interface TransactionStripped {

17
backend/src/repositories/BlocksRepository.ts
View file

@ -662,6 +662,23 @@ class BlocksRepository {
}
}
/**
* Get a list of blocks that have not had CPFP data indexed
*/
public async $getCPFPUnindexedBlocks(): Promise<any[]> {
try {
const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`);
return rows;
} catch (e) {
logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $setCPFPIndexed(hash: string): Promise<void> {
await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]);
}
/**
* Return the oldest block from a consecutive chain of block from the most recent one
*/

43
backend/src/repositories/CpfpRepository.ts Normal file
View file

@ -0,0 +1,43 @@
import DB from '../database';
import logger from '../logger';
import { Ancestor } from '../mempool.interfaces';
class CpfpRepository {
public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
try {
const txsJson = JSON.stringify(txs);
await DB.query(
`
INSERT INTO cpfp_clusters(root, height, txs, fee_rate)
VALUE (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
height = ?,
txs = ?,
fee_rate = ?
`,
[txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height]
);
} catch (e: any) {
logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $deleteClustersFrom(height: number): Promise<void> {
logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
try {
await DB.query(
`
DELETE from cpfp_clusters
WHERE height >= ?
`,
[height]
);
} catch (e: any) {
logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new CpfpRepository();

77
backend/src/repositories/TransactionRepository.ts Normal file
View file

@ -0,0 +1,77 @@
import DB from '../database';
import logger from '../logger';
import { Ancestor, CpfpInfo } from '../mempool.interfaces';
interface CpfpSummary {
txid: string;
cluster: string;
root: string;
txs: Ancestor[];
height: number;
fee_rate: number;
}
class TransactionRepository {
public async $setCluster(txid: string, cluster: string): Promise<void> {
try {
await DB.query(
`
INSERT INTO transactions
(
txid,
cluster
)
VALUE (?, ?)
ON DUPLICATE KEY UPDATE
cluster = ?
;`,
[txid, cluster, cluster]
);
} catch (e: any) {
logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
try {
let query = `
SELECT *
FROM transactions
LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster
WHERE transactions.txid = ?
`;
const [rows]: any = await DB.query(query, [txid]);
if (rows.length) {
rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[];
return this.convertCpfp(rows[0]);
}
} catch (e) {
logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
private convertCpfp(cpfp: CpfpSummary): CpfpInfo {
const descendants: Ancestor[] = [];
const ancestors: Ancestor[] = [];
let matched = false;
for (const tx of cpfp.txs) {
if (tx.txid === cpfp.txid) {
matched = true;
} else if (!matched) {
descendants.push(tx);
} else {
ancestors.push(tx);
}
}
return {
descendants,
ancestors,
effectiveFeePerVsize: cpfp.fee_rate
};
}
}
export default new TransactionRepository();