mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-11-20 02:11:49 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into update_gha

Browse Source
This commit is contained in:
Felipe Knorr Kuhn 2022-12-07 19:06:48 -08:00 committed by GitHub
commit 250df0d56c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 98 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
.github/dependabot.yml vendored
View File

@ -1,20 +1,28 @@
version: 2
updates:
- package-ecosystem: npm
directory: "/backend"
schedule:
interval: daily
open-pull-requests-limit: 10
- package-ecosystem: npm
directory: "/frontend"
schedule:
interval: daily
open-pull-requests-limit: 10
- package-ecosystem: docker
directory: "/docker/backend"
schedule:
interval: weekly
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: npm
directory: "/backend"
schedule:
interval: daily
open-pull-requests-limit: 10
ignore:
- update-types: ["version-update:semver-major"]
- package-ecosystem: npm
directory: "/frontend"
schedule:
interval: daily
open-pull-requests-limit: 10
ignore:
- update-types: ["version-update:semver-major"]
- package-ecosystem: docker
directory: "/docker/backend"
schedule:
interval: daily
ignore:
- update-types: ["version-update:semver-major"]
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: daily
ignore:
- update-types: ["version-update:semver-major"]

3
backend/mempool-config.sample.json
View File

@ -85,7 +85,8 @@
"STATS_REFRESH_INTERVAL": 600,
"GRAPH_REFRESH_INTERVAL": 600,
"LOGGER_UPDATE_INTERVAL": 30,
"FORENSICS_INTERVAL": 43200
"FORENSICS_INTERVAL": 43200,
"FORENSICS_RATE_LIMIT": 20
},
"LND": {
"TLS_CERT_PATH": "tls.cert",

3
backend/src/__fixtures__/mempool-config.template.json
View File

@ -101,7 +101,8 @@
"STATS_REFRESH_INTERVAL": 600,
"GRAPH_REFRESH_INTERVAL": 600,
"LOGGER_UPDATE_INTERVAL": 30,
"FORENSICS_INTERVAL": 43200
"FORENSICS_INTERVAL": 43200,
"FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__"
},
"LND": {
"TLS_CERT_PATH": "",

4
backend/src/api/explorer/nodes.api.ts
View File

@ -538,6 +538,10 @@ class NodesApi {
const IPSIds = ISPId.split(',');
const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
if (!rows || rows.length === 0) {
return [];
}
const nodes = {};
const intISPIds: number[] = [];

54
backend/src/api/tx-selection-worker.ts
View File

@ -106,44 +106,44 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }:
if (nextTx && !nextTx?.used) {
// Check if the package fits into this block
if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
blockWeight += nextTx.ancestorWeight;
const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
const descendants: AuditTransaction[] = [];
// sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
const used: AuditTransaction[] = [];
while (sortedTxSet.length) {
const ancestor = sortedTxSet.pop();
const mempoolTx = mempool[ancestor.txid];
if (ancestor && !ancestor?.used) {
ancestor.used = true;
// update original copy of this tx with effective fee rate & relatives data
mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
mempoolTx.ancestors = sortedTxSet.map((a) => {
return {
txid: a.txid,
fee: a.fee,
weight: a.weight,
};
}).reverse();
mempoolTx.descendants = descendants.map((a) => {
return {
txid: a.txid,
fee: a.fee,
weight: a.weight,
};
});
descendants.push(ancestor);
mempoolTx.cpfpChecked = true;
transactions.push(ancestor);
blockSize += ancestor.size;
}
ancestor.used = true;
ancestor.usedBy = nextTx.txid;
// update original copy of this tx with effective fee rate & relatives data
mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
mempoolTx.ancestors = sortedTxSet.map((a) => {
return {
txid: a.txid,
fee: a.fee,
weight: a.weight,
};
}).reverse();
mempoolTx.descendants = descendants.map((a) => {
return {
txid: a.txid,
fee: a.fee,
weight: a.weight,
};
});
descendants.push(ancestor);
mempoolTx.cpfpChecked = true;
transactions.push(ancestor);
blockSize += ancestor.size;
blockWeight += ancestor.weight;
used.push(ancestor);
}
// remove these as valid package ancestors for any descendants remaining in the mempool
if (sortedTxSet.length) {
sortedTxSet.forEach(tx => {
if (used.length) {
used.forEach(tx => {
updateDescendants(tx, auditPool, modified);
});
}

4
backend/src/api/websocket-handler.ts
View File

@ -439,7 +439,7 @@ class WebsocketHandler {
};
}) : [];
BlocksSummariesRepository.$saveSummary({
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
@ -468,7 +468,7 @@ class WebsocketHandler {
}
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.makeBlockTemplates(_memPool, 2);
await mempoolBlocks.makeBlockTemplates(_memPool, 8, null, true);
} else {
mempoolBlocks.updateMempoolBlocks(_memPool);
}

2
backend/src/config.ts
View File

@ -44,6 +44,7 @@ interface IConfig {
GRAPH_REFRESH_INTERVAL: number;
LOGGER_UPDATE_INTERVAL: number;
FORENSICS_INTERVAL: number;
FORENSICS_RATE_LIMIT: number;
};
LND: {
TLS_CERT_PATH: string;
@ -205,6 +206,7 @@ const defaults: IConfig = {
'GRAPH_REFRESH_INTERVAL': 600,
'LOGGER_UPDATE_INTERVAL': 30,
'FORENSICS_INTERVAL': 43200,
'FORENSICS_RATE_LIMIT': 20,
},
'LND': {
'TLS_CERT_PATH': '',

41
backend/src/repositories/BlocksSummariesRepository.ts
View File

@ -17,19 +17,16 @@ class BlocksSummariesRepository {
return undefined;
}
public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) {
const blockId = params.mined?.id ?? params.template?.id;
public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
const blockId = params.mined?.id;
try {
const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`);
if (dbSummary.length === 0) { // First insertion
await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [
params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? [])
]);
} else if (params.mined !== undefined) { // Update mined block summary
await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]);
} else if (params.template !== undefined) { // Update template block summary
await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]);
}
const transactions = JSON.stringify(params.mined?.transactions || []);
await DB.query(`
INSERT INTO blocks_summaries (height, id, transactions, template)
VALUE (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
transactions = ?
`, [params.height, blockId, transactions, '[]', transactions]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
@ -40,6 +37,26 @@ class BlocksSummariesRepository {
}
}
public async $saveTemplate(params: { height: number, template: BlockSummary}) {
const blockId = params.template?.id;
try {
const transactions = JSON.stringify(params.template?.transactions || []);
await DB.query(`
INSERT INTO blocks_summaries (height, id, transactions, template)
VALUE (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
template = ?
`, [params.height, blockId, '[]', transactions, transactions]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
} else {
logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
}
public async $getIndexedSummariesId(): Promise<string[]> {
try {
const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);

7
backend/src/tasks/lightning/forensics.service.ts
View File

@ -7,7 +7,6 @@ import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import { Common } from '../../api/common';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
const throttleDelay = 20; //ms
const tempCacheSize = 10000;
class ForensicsService {
@ -91,7 +90,7 @@ class ForensicsService {
let outspends: IEsploraApi.Outspend[] | undefined;
try {
outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
await Common.sleep$(throttleDelay);
await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
continue;
@ -340,7 +339,7 @@ class ForensicsService {
let outspends: IEsploraApi.Outspend[] | undefined;
try {
outspends = await bitcoinApi.$getOutspends(input.txid);
await Common.sleep$(throttleDelay);
await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
}
@ -429,7 +428,7 @@ class ForensicsService {
if (temp) {
this.tempCached.push(txid);
}
await Common.sleep$(throttleDelay);
await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
} catch (e) {
logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
return null;

2
frontend/proxy.conf.staging.js
View File

@ -5,7 +5,7 @@ let PROXY_CONFIG = require('./proxy.conf');
PROXY_CONFIG.forEach(entry => {
entry.target = entry.target.replace("mempool.space", "mempool-staging.tk7.mempool.space");
entry.target = entry.target.replace("liquid.network", "liquid-staging.tk7.mempool.space");
entry.target = entry.target.replace("bisq.markets", "bisq-staging.tk7.mempool.space");
entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space");
});
module.exports = PROXY_CONFIG;