Merge pull request #3846 from mempool/mononaut/audit-details

Add expected vs actual audit details comparison table
This commit is contained in:
softsimon 2023-06-14 23:05:07 +02:00 committed by GitHub
commit c5bf167e36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 282 additions and 18 deletions

View File

@ -282,10 +282,14 @@ class Blocks {
}
extras.matchRate = null;
extras.expectedFees = null;
extras.expectedWeight = null;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
extras.matchRate = auditScore.matchRate;
extras.expectedFees = auditScore.expectedFees;
extras.expectedWeight = auditScore.expectedWeight;
}
}
}
@ -455,6 +459,46 @@ class Blocks {
}
}
/**
* [INDEXING] Index expected fees & weight for all audited blocks
*/
public async $generateAuditStats(): Promise<void> {
const blockIds = await BlocksAuditsRepository.$getBlocksWithoutSummaries();
if (!blockIds?.length) {
return;
}
let timer = Date.now();
let indexedThisRun = 0;
let indexedTotal = 0;
logger.debug(`Indexing ${blockIds.length} block audit details`);
for (const hash of blockIds) {
const summary = await BlocksSummariesRepository.$getTemplate(hash);
let totalFees = 0;
let totalWeight = 0;
for (const tx of summary?.transactions || []) {
totalFees += tx.fee;
totalWeight += (tx.vsize * 4);
}
await BlocksAuditsRepository.$setSummary(hash, totalFees, totalWeight);
const cachedBlock = this.blocks.find(block => block.id === hash);
if (cachedBlock) {
cachedBlock.extras.expectedFees = totalFees;
cachedBlock.extras.expectedWeight = totalWeight;
}
indexedThisRun++;
indexedTotal++;
const elapsedSeconds = (Date.now() - timer) / 1000;
if (elapsedSeconds > 5) {
const blockPerSeconds = indexedThisRun / elapsedSeconds;
logger.debug(`Indexed ${indexedTotal} / ${blockIds.length} block audit details (${blockPerSeconds.toFixed(1)}/s)`);
timer = Date.now();
indexedThisRun = 0;
}
}
logger.debug(`Indexing block audit details completed`);
}
/**
* [INDEXING] Index all blocks metadata for the mining dashboard
*/

View File

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 61;
private static currentVersion = 62;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -533,6 +533,12 @@ class DatabaseMigration {
await this.updateToSchemaVersion(61);
}
if (databaseSchemaVersion < 62 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_weight BIGINT UNSIGNED DEFAULT NULL');
await this.updateToSchemaVersion(62);
}
}
/**

View File

@ -571,11 +571,18 @@ class WebsocketHandler {
};
}) : [];
let totalFees = 0;
let totalWeight = 0;
for (const tx of stripped) {
totalFees += tx.fee;
totalWeight += (tx.vsize * 4);
}
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
transactions: stripped
transactions: stripped,
}
});
@ -588,10 +595,14 @@ class WebsocketHandler {
freshTxs: fresh,
sigopTxs: sigop,
matchRate: matchRate,
expectedFees: totalFees,
expectedWeight: totalWeight,
});
if (block.extras) {
block.extras.matchRate = matchRate;
block.extras.expectedFees = totalFees;
block.extras.expectedWeight = totalWeight;
block.extras.similarity = similarity;
}
}

View File

@ -134,6 +134,7 @@ class Indexer {
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();
await blocks.$generateCPFPDatabase();
await blocks.$generateAuditStats();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

View File

@ -35,11 +35,15 @@ export interface BlockAudit {
sigopTxs: string[],
addedTxs: string[],
matchRate: number,
expectedFees?: number,
expectedWeight?: number,
}
export interface AuditScore {
hash: string,
matchRate?: number,
expectedFees?: number
expectedWeight?: number
}
export interface MempoolBlock {
@ -182,6 +186,8 @@ export interface BlockExtension {
feeRange: number[]; // fee rate percentiles
reward: number;
matchRate: number | null;
expectedFees: number | null;
expectedWeight: number | null;
similarity?: number;
pool: {
id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`

View File

@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces';
class BlocksAuditRepositories {
public async $saveAudit(audit: BlockAudit): Promise<void> {
try {
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate]);
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@ -18,6 +18,19 @@ class BlocksAuditRepositories {
}
}
public async $setSummary(hash: string, expectedFees: number, expectedWeight: number) {
try {
await DB.query(`
UPDATE blocks_audits SET
expected_fees = ?,
expected_weight = ?
WHERE hash = ?
`, [expectedFees, expectedWeight, hash]);
} catch (e: any) {
logger.err(`Cannot update block audit in db. Reason: ` + (e instanceof Error ? e.message : e));
}
}
public async $getBlocksHealthHistory(div: number, interval: string | null): Promise<any> {
try {
let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
@ -51,7 +64,15 @@ class BlocksAuditRepositories {
const [rows]: any[] = await DB.query(
`SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
blocks.weight, blocks.tx_count,
transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, match_rate as matchRate
transactions,
template,
missing_txs as missingTxs,
added_txs as addedTxs,
fresh_txs as freshTxs,
sigop_txs as sigopTxs,
match_rate as matchRate,
expected_fees as expectedFees,
expected_weight as expectedWeight
FROM blocks_audits
JOIN blocks ON blocks.hash = blocks_audits.hash
JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
@ -81,7 +102,7 @@ class BlocksAuditRepositories {
public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
try {
const [rows]: any[] = await DB.query(
`SELECT hash, match_rate as matchRate
`SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight
FROM blocks_audits
WHERE blocks_audits.hash = "${hash}"
`);
@ -95,7 +116,7 @@ class BlocksAuditRepositories {
public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise<AuditScore[]> {
try {
const [rows]: any[] = await DB.query(
`SELECT hash, match_rate as matchRate
`SELECT hash, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight
FROM blocks_audits
WHERE blocks_audits.height BETWEEN ? AND ?
`, [minHeight, maxHeight]);
@ -105,6 +126,32 @@ class BlocksAuditRepositories {
throw e;
}
}
public async $getBlocksWithoutSummaries(): Promise<string[]> {
try {
const [fromRows]: any[] = await DB.query(`
SELECT height
FROM blocks_audits
WHERE expected_fees IS NULL
ORDER BY height DESC
LIMIT 1
`);
if (!fromRows?.length) {
return [];
}
const fromHeight = fromRows[0].height;
const [idRows]: any[] = await DB.query(`
SELECT hash
FROM blocks_audits
WHERE height <= ?
ORDER BY height DESC
`, [fromHeight]);
return idRows.map(row => row.hash);
} catch (e: any) {
logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new BlocksAuditRepositories();

View File

@ -1018,10 +1018,14 @@ class BlocksRepository {
// Match rate is not part of the blocks table, but it is part of APIs so we must include it
extras.matchRate = null;
extras.expectedFees = null;
extras.expectedWeight = null;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id);
if (auditScore != null) {
extras.matchRate = auditScore.matchRate;
extras.expectedFees = auditScore.expectedFees;
extras.expectedWeight = auditScore.expectedWeight;
}
}

View File

@ -50,6 +50,21 @@ class BlocksSummariesRepository {
}
}
public async $getTemplate(id: string): Promise<BlockSummary | undefined> {
try {
const [templates]: any[] = await DB.query(`SELECT * from blocks_templates WHERE id = ?`, [id]);
if (templates.length > 0) {
return {
id: templates[0].id,
transactions: JSON.parse(templates[0].template),
};
}
} catch (e) {
logger.err(`Cannot get block template for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e));
}
return undefined;
}
public async $getIndexedSummariesId(): Promise<string[]> {
try {
const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);

View File

@ -0,0 +1,3 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
Signed: joostjager

View File

@ -226,6 +226,9 @@
(txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)" [unavailable]="!isMobile && !showAudit"></app-block-overview-graph>
<ng-container *ngIf="!isMobile || mode !== 'actual'; else emptyBlockInfo"></ng-container>
</div>
<ng-container *ngIf="network !== 'liquid'">
<ng-container *ngTemplateOutlet="isMobile && mode === 'actual' ? actualDetails : expectedDetails"></ng-container>
</ng-container>
</div>
<div class="col-sm" *ngIf="!isMobile">
<h3 class="block-subtitle actual" *ngIf="!isMobile"><ng-container i18n="block.actual-block">Actual Block</ng-container> <a class="info-link" [routerLink]="['/docs/faq' | relativeUrl ]" fragment="how-do-block-audits-work"><fa-icon [icon]="['fas', 'info-circle']" [fixedWidth]="true"></fa-icon></a></h3>
@ -235,6 +238,9 @@
(txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)" [unavailable]="isMobile && !showAudit"></app-block-overview-graph>
<ng-container *ngTemplateOutlet="emptyBlockInfo"></ng-container>
</div>
<ng-container *ngIf="network !== 'liquid'">
<ng-container *ngTemplateOutlet="actualDetails"></ng-container>
</ng-container>
</div>
</div>
</div>
@ -385,5 +391,60 @@
</a>
</ng-template>
<ng-template #expectedDetails>
<table *ngIf="block && blockAudit && blockAudit.expectedFees != null" class="table table-borderless table-striped audit-details-table">
<tbody>
<tr>
<td i18n="block.total-fees|Total fees in a block">Total fees</td>
<td>
<app-amount [satoshis]="blockAudit.expectedFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
</td>
</tr>
<tr>
<td i18n="block.weight">Weight</td>
<td [innerHTML]="'&lrm;' + (blockAudit.expectedWeight | wuBytes: 2)"></td>
</tr>
<tr>
<td i18n="mempool-block.transactions">Transactions</td>
<td>{{ blockAudit.template?.length || 0 }}</td>
</tr>
</tbody>
</table>
</ng-template>
<ng-template #actualDetails>
<table *ngIf="block && blockAudit && blockAudit.expectedFees != null" class="table table-borderless table-striped audit-details-table">
<tbody>
<tr>
<td i18n="block.total-fees|Total fees in a block">Total fees</td>
<td>
<app-amount [satoshis]="block.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
<span *ngIf="blockAudit.feeDelta" class="difference" [class.positive]="blockAudit.feeDelta <= 0" [class.negative]="blockAudit.feeDelta > 0">
{{ blockAudit.feeDelta < 0 ? '+' : '' }}{{ (-blockAudit.feeDelta * 100) | amountShortener: 2 }}%
</span>
</td>
</tr>
<tr>
<td i18n="block.weight">Weight</td>
<td [innerHTML]>
<span [innerHTML]="'&lrm;' + (block.weight | wuBytes: 2)"></span>
<span *ngIf="blockAudit.weightDelta" class="difference" [class.positive]="blockAudit.weightDelta <= 0" [class.negative]="blockAudit.weightDelta > 0">
{{ blockAudit.weightDelta < 0 ? '+' : '' }}{{ (-blockAudit.weightDelta * 100) | amountShortener: 2 }}%
</span>
</td>
</tr>
<tr>
<td i18n="mempool-block.transactions">Transactions</td>
<td>
{{ block.tx_count }}
<span *ngIf="blockAudit.txDelta" class="difference" [class.positive]="blockAudit.txDelta <= 0" [class.negative]="blockAudit.txDelta > 0">
{{ blockAudit.txDelta < 0 ? '+' : '' }}{{ (-blockAudit.txDelta * 100) | amountShortener: 2 }}%
</span>
</td>
</tr>
</tbody>
</table>
</ng-template>
<br>
<br>

View File

@ -38,6 +38,17 @@
color: rgba(255, 255, 255, 0.4);
margin-left: 5px;
}
.difference {
margin-left: 0.5em;
&.positive {
color: rgb(66, 183, 71);
}
&.negative {
color: rgb(183, 66, 66);
}
}
}
}
@ -252,3 +263,10 @@ h1 {
top: 11px;
margin-left: 10px;
}
.audit-details-table {
margin-top: 1.25rem;
@media (max-width: 767.98px) {
margin-top: 0.75rem;
}
}

View File

@ -388,6 +388,11 @@ export class BlockComponent implements OnInit, OnDestroy {
for (const tx of blockAudit.transactions) {
inBlock[tx.txid] = true;
}
blockAudit.feeDelta = blockAudit.expectedFees > 0 ? (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees : 0;
blockAudit.weightDelta = blockAudit.expectedWeight > 0 ? (blockAudit.expectedWeight - this.block.weight) / blockAudit.expectedWeight : 0;
blockAudit.txDelta = blockAudit.template.length > 0 ? (blockAudit.template.length - this.block.tx_count) / blockAudit.template.length : 0;
this.setAuditAvailable(true);
} else {
this.setAuditAvailable(false);

View File

@ -13,12 +13,12 @@
<th *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}" i18n="mining.pool-name"
i18n-ngbTooltip="mining.pool-name" ngbTooltip="Pool" placement="bottom" #miningpool [disableTooltip]="!isEllipsisActive(miningpool)">Pool</th>
<th class="timestamp" i18n="latest-blocks.timestamp" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">Timestamp</th>
<th class="mined" i18n="latest-blocks.mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">Mined</th>
<th *ngIf="auditAvailable" class="health text-right" i18n="latest-blocks.health" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
i18n-ngbTooltip="latest-blocks.health" ngbTooltip="Health" placement="bottom" #health [disableTooltip]="!isEllipsisActive(health)">Health</th>
<th *ngIf="indexingAvailable" class="reward text-right" i18n="latest-blocks.reward" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
i18n-ngbTooltip="latest-blocks.reward" ngbTooltip="Reward" placement="bottom" #reward [disableTooltip]="!isEllipsisActive(reward)">Reward</th>
<th *ngIf="indexingAvailable && !widget" class="fees text-right" i18n="latest-blocks.fees" [class]="indexingAvailable ? '' : 'legacy'">Fees</th>
<th *ngIf="auditAvailable && !widget" class="fee-delta" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"></th>
<th *ngIf="indexingAvailable" class="txs text-right" i18n="dashboard.txs" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
i18n-ngbTooltip="dashboard.txs" ngbTooltip="TXs" placement="bottom" #txs [disableTooltip]="!isEllipsisActive(txs)">TXs</th>
<th *ngIf="!indexingAvailable" class="txs text-right" i18n="dashboard.txs" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">Transactions</th>
@ -42,9 +42,6 @@
<td class="timestamp" *ngIf="!widget" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
&lrm;{{ block.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }}
</td>
<td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
<app-time kind="since" [time]="block.timestamp" [fastRender]="true"></app-time>
</td>
<td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<a
class="health-badge badge"
@ -70,6 +67,11 @@
<td *ngIf="indexingAvailable && !widget" class="fees text-right" [class]="indexingAvailable ? '' : 'legacy'">
<app-amount [satoshis]="block.extras.totalFees" [noFiat]="true" digitsInfo="1.2-2"></app-amount>
</td>
<td *ngIf="auditAvailable" class="fee-delta" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<span *ngIf="block.extras.feeDelta" class="difference" [class.positive]="block.extras.feeDelta >= 0" [class.negative]="block.extras.feeDelta < 0">
{{ block.extras.feeDelta > 0 ? '+' : '' }}{{ (block.extras.feeDelta * 100) | amountShortener: 2 }}%
</span>
</td>
<td class="txs text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
{{ block.tx_count | number }}
</td>
@ -106,6 +108,9 @@
<td *ngIf="indexingAvailable && !widget" class="fees text-right" [class]="indexingAvailable ? '' : 'legacy'">
<span class="skeleton-loader" style="max-width: 75px"></span>
</td>
<td *ngIf="auditAvailable && !widget" class="fee-delta" [class]="indexingAvailable ? '' : 'legacy'">
<span class="skeleton-loader" style="max-width: 75px"></span>
</td>
<td class="txs text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<span class="skeleton-loader" style="max-width: 75px"></span>
</td>

View File

@ -23,6 +23,17 @@ tr, td, th {
border: 0px;
padding-top: 0.65rem !important;
padding-bottom: 0.7rem !important;
.difference {
margin-left: 0.5em;
&.positive {
color: rgb(66, 183, 71);
}
&.negative {
color: rgb(183, 66, 66);
}
}
}
.clear-link {
@ -90,7 +101,7 @@ tr, td, th {
}
.timestamp {
width: 18%;
width: 10%;
@media (max-width: 1100px) {
display: none;
}
@ -123,8 +134,8 @@ tr, td, th {
}
.txs {
padding-right: 40px;
width: 8%;
padding-right: 20px;
width: 6%;
@media (max-width: 1100px) {
padding-right: 10px;
}
@ -160,6 +171,16 @@ tr, td, th {
.fees.widget {
width: 20%;
}
.fee-delta {
width: 6%;
padding-left: 0;
@media (max-width: 991px) {
display: none;
}
}
.fee-delta.widget {
display: none;
}
.reward {
width: 8%;
@ -214,7 +235,7 @@ tr, td, th {
.health {
width: 10%;
@media (max-width: 1105px) {
@media (max-width: 1100px) {
width: 13%;
}
@media (max-width: 560px) {

View File

@ -1,4 +1,4 @@
import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input } from '@angular/core';
import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input, ChangeDetectorRef } from '@angular/core';
import { BehaviorSubject, combineLatest, concat, Observable, timer, EMPTY, Subscription, of } from 'rxjs';
import { catchError, delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators';
import { BlockExtended } from '../../interfaces/node-api.interface';
@ -39,6 +39,7 @@ export class BlocksList implements OnInit, OnDestroy {
private apiService: ApiService,
private websocketService: WebsocketService,
public stateService: StateService,
private cd: ChangeDetectorRef,
) {
}
@ -112,7 +113,13 @@ export class BlocksList implements OnInit, OnDestroy {
acc = acc.slice(0, this.widget ? 6 : 15);
}
return acc;
}, [])
}, []),
switchMap((blocks) => {
blocks.forEach(block => {
block.extras.feeDelta = block.extras.expectedFees ? (block.extras.totalFees - block.extras.expectedFees) / block.extras.expectedFees : 0;
});
return of(blocks);
})
);
if (this.indexingAvailable && this.auditAvailable) {
@ -131,6 +138,7 @@ export class BlocksList implements OnInit, OnDestroy {
this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null;
});
this.loadingScores = false;
this.cd.markForCheck();
});
this.latestScoreSubscription = this.stateService.blocks$.pipe(
@ -155,6 +163,7 @@ export class BlocksList implements OnInit, OnDestroy {
).subscribe((score) => {
if (score && score.hash) {
this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null;
this.cd.markForCheck();
}
});
}

View File

@ -133,6 +133,9 @@ export interface BlockExtension {
reward?: number;
coinbaseRaw?: string;
matchRate?: number;
expectedFees?: number;
expectedWeight?: number;
feeDelta?: number;
similarity?: number;
pool?: {
id: number;
@ -149,6 +152,11 @@ export interface BlockAudit extends BlockExtended {
missingTxs: string[],
addedTxs: string[],
matchRate: number,
expectedFees: number,
expectedWeight: number,
feeDelta?: number,
weightDelta?: number,
txDelta?: number,
template: TransactionStripped[],
transactions: TransactionStripped[],
}