Merge pull request #5460 from mempool/mononaut/v1-audit-improvements

v1 audit improvements

backend/src/api/audit.ts

@@ -2,6 +2,7 @@ import config from '../config';
 import logger from '../logger';
 import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 import rbfCache from './rbf-cache';
+import transactionUtils from './transaction-utils';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
@@ -15,7 +16,8 @@ class Audit {
     const matches: string[] = []; // present in both mined block and template
     const added: string[] = []; // present in mined block, not in template
     const unseen: string[] = []; // present in the mined block, not in our mempool
-    const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
+    let prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
+    let deprioritized: string[] = []; // lower in the block than would be expected by in-band feerate alone
     const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
     const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
     const accelerated: string[] = []; // prioritized by the mempool accelerator
@@ -133,23 +135,7 @@ class Audit {
       totalWeight += tx.weight;
     }
 
-
-    // identify "prioritized" transactions
-    let lastEffectiveRate = 0;
-    // Iterate over the mined template from bottom to top (excluding the coinbase)
-    // Transactions should appear in ascending order of mining priority.
-    for (let i = transactions.length - 1; i > 0; i--) {
-      const blockTx = transactions[i];
-      // If a tx has a lower in-band effective fee rate than the previous tx,
-      // it must have been prioritized out-of-band (in order to have a higher mining priority)
-      // so exclude from the analysis.
-      if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) {
-        prioritized.push(blockTx.txid);
-        // accelerated txs may or may not have their prioritized fee rate applied, so don't use them as a reference
-      } else if (!isAccelerated[blockTx.txid]) {
-        lastEffectiveRate = blockTx.effectiveFeePerVsize || 0;
-      }
-    }
+    ({ prioritized, deprioritized } = transactionUtils.identifyPrioritizedTransactions(transactions, 'effectiveFeePerVsize'));
 
     // transactions missing from near the end of our template are probably not being censored
     let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 81;
+  private static currentVersion = 82;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -700,6 +700,11 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `blocks_audits` ADD unseen_txs JSON DEFAULT "[]"');
       await this.updateToSchemaVersion(81);
     }
+
+    if (databaseSchemaVersion < 82 && isBitcoin === true && config.MEMPOOL.NETWORK === 'mainnet') {
+      await this.$fixBadV1AuditBlocks();
+      await this.updateToSchemaVersion(82);
+    }
   }
 
   /**
@@ -1314,6 +1319,28 @@ class DatabaseMigration {
       logger.warn(`Failed to migrate cpfp transaction data`);
     }
   }
+
+  private async $fixBadV1AuditBlocks(): Promise<void> {
+    const badBlocks = [
+      '000000000000000000011ad49227fc8c9ba0ca96ad2ebce41a862f9a244478dc',
+      '000000000000000000010ac1f68b3080153f2826ffddc87ceffdd68ed97d6960',
+      '000000000000000000024cbdafeb2660ae8bd2947d166e7fe15d1689e86b2cf7',
+      '00000000000000000002e1dbfbf6ae057f331992a058b822644b368034f87286',
+      '0000000000000000000019973b2778f08ad6d21e083302ff0833d17066921ebb',
+    ];
+
+    for (const hash of badBlocks) {
+      try {
+        await this.$executeQuery(`
+          UPDATE blocks_audits
+          SET prioritized_txs = '[]'
+          WHERE hash = '${hash}'
+        `, true);
+      } catch (e) {
+        continue;
+      }
+    }
+  }
 }
 
 export default new DatabaseMigration();

backend/src/api/transaction-utils.ts

@@ -338,6 +338,87 @@ class TransactionUtils {
     const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
     return witness[positionOfScript];
   }
+
+  // calculate the most parsimonious set of prioritizations given a list of block transactions
+  // (i.e. the most likely prioritizations and deprioritizations)
+  public identifyPrioritizedTransactions(transactions: any[], rateKey: string): { prioritized: string[], deprioritized: string[] } {
+    // find the longest increasing subsequence of transactions
+    // (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms)
+    // should be O(n log n)
+    const X = transactions.slice(1).reverse().map((tx) => ({ txid: tx.txid, rate: tx[rateKey] })); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase)
+    if (X.length < 2) {
+      return { prioritized: [], deprioritized: [] };
+    }
+    const N = X.length;
+    const P: number[] = new Array(N);
+    const M: number[] = new Array(N + 1);
+    M[0] = -1; // undefined so can be set to any value
+
+    let L = 0;
+    for (let i = 0; i < N; i++) {
+      // Binary search for the smallest positive l ≤ L
+      // such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize
+      let lo = 1;
+      let hi = L + 1;
+      while (lo < hi) {
+        const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi
+        if (X[M[mid]].rate > X[i].rate) {
+          hi = mid;
+        } else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize
+          lo = mid + 1;
+        }
+      }
+
+      // After searching, lo == hi is 1 greater than the
+      // length of the longest prefix of X[i]
+      const newL = lo;
+
+      // The predecessor of X[i] is the last index of
+      // the subsequence of length newL-1
+      P[i] = M[newL - 1];
+      M[newL] = i;
+
+      if (newL > L) {
+        // If we found a subsequence longer than any we've
+        // found yet, update L
+        L = newL;
+      }
+    }
+
+    // Reconstruct the longest increasing subsequence
+    // It consists of the values of X at the L indices:
+    // ..., P[P[M[L]]], P[M[L]], M[L]
+    const LIS: any[] = new Array(L);
+    let k = M[L];
+    for (let j = L - 1; j >= 0; j--) {
+      LIS[j] = X[k];
+      k = P[k];
+    }
+
+    const lisMap = new Map<string, number>();
+    LIS.forEach((tx, index) => lisMap.set(tx.txid, index));
+
+    const prioritized: string[] = [];
+    const deprioritized: string[] = [];
+
+    let lastRate = X[0].rate;
+
+    for (const tx of X) {
+      if (lisMap.has(tx.txid)) {
+        lastRate = tx.rate;
+      } else {
+        if (Math.abs(tx.rate - lastRate) < 0.1) {
+          // skip if the rate is almost the same as the previous transaction
+        } else if (tx.rate <= lastRate) {
+          prioritized.push(tx.txid);
+        } else {
+          deprioritized.push(tx.txid);
+        }
+      }
+    }
+
+    return { prioritized, deprioritized };
+  }
 }
 
 export default new TransactionUtils();

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -33,7 +33,7 @@ export default class TxView implements TransactionStripped {
   flags: number;
   bigintFlags?: bigint | null = 0b00000100_00000000_00000000_00000000n;
   time?: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'added_deprioritized' | 'deprioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
   context?: 'projected' | 'actual';
   scene?: BlockScene;
 

frontend/src/app/components/block-overview-graph/utils.ts

@@ -142,6 +142,10 @@ export function defaultColorFunction(
       return auditColors.added_prioritized;
     case 'prioritized':
       return auditColors.prioritized;
+    case 'added_deprioritized':
+      return auditColors.added_prioritized;
+    case 'deprioritized':
+      return auditColors.prioritized;
     case 'selected':
       return colors.marginal[levelIndex] || colors.marginal[defaultMempoolFeeColors.length - 1];
     case 'accelerated':

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -79,6 +79,11 @@
               <span class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
               <span class="badge badge-warning ml-1" i18n="tx-features.tag.prioritized|Prioritized">Prioritized</span>
             </ng-container>
+            <span *ngSwitchCase="'deprioritized'" class="badge badge-warning" i18n="tx-features.tag.prioritized|Deprioritized">Deprioritized</span>
+            <ng-container *ngSwitchCase="'added_deprioritized'">
+              <span class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
+              <span class="badge badge-warning ml-1" i18n="tx-features.tag.prioritized|Deprioritized">Deprioritized</span>
+            </ng-container>
             <span *ngSwitchCase="'selected'" class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span>
             <span *ngSwitchCase="'rbf'" class="badge badge-warning" i18n="tx-features.tag.conflict|Conflict">Conflict</span>
             <span *ngSwitchCase="'accelerated'" class="badge badge-accelerated" i18n="transaction.audit.accelerated">Accelerated</span>

frontend/src/app/components/block/block.component.ts

@@ -17,6 +17,7 @@ import { PriceService, Price } from '../../services/price.service';
 import { CacheService } from '../../services/cache.service';
 import { ServicesApiServices } from '../../services/services-api.service';
 import { PreloadService } from '../../services/preload.service';
+import { identifyPrioritizedTransactions } from '../../shared/transaction.utils';
 
 @Component({
   selector: 'app-block',
@@ -524,6 +525,7 @@ export class BlockComponent implements OnInit, OnDestroy {
       const isUnseen = {};
       const isAdded = {};
       const isPrioritized = {};
+      const isDeprioritized = {};
       const isCensored = {};
       const isMissing = {};
       const isSelected = {};
@@ -535,6 +537,17 @@ export class BlockComponent implements OnInit, OnDestroy {
       this.numUnexpected = 0;
 
       if (blockAudit?.template) {
+        // augment with locally calculated *de*prioritized transactions if possible
+        const { prioritized, deprioritized } = identifyPrioritizedTransactions(transactions);
+        // but if the local calculation produces returns unexpected results, don't use it
+        let useLocalDeprioritized = deprioritized.length < (transactions.length * 0.1);
+        for (const tx of prioritized) {
+          if (!isPrioritized[tx] && !isAccelerated[tx]) {
+            useLocalDeprioritized = false;
+            break;
+          }
+        }
+
         for (const tx of blockAudit.template) {
           inTemplate[tx.txid] = true;
           if (tx.acc) {
@@ -550,9 +563,14 @@ export class BlockComponent implements OnInit, OnDestroy {
         for (const txid of blockAudit.addedTxs) {
           isAdded[txid] = true;
         }
-        for (const txid of blockAudit.prioritizedTxs || []) {
+        for (const txid of blockAudit.prioritizedTxs) {
           isPrioritized[txid] = true;
         }
+        if (useLocalDeprioritized) {
+          for (const txid of deprioritized || []) {
+            isDeprioritized[txid] = true;
+          }
+        }
         for (const txid of blockAudit.missingTxs) {
           isCensored[txid] = true;
         }
@@ -608,6 +626,12 @@ export class BlockComponent implements OnInit, OnDestroy {
             } else {
               tx.status = 'prioritized';
             }
+          } else if (isDeprioritized[tx.txid]) {
+            if (isAdded[tx.txid] || (blockAudit.version > 0 && isUnseen[tx.txid])) {
+              tx.status = 'added_deprioritized';
+            } else {
+              tx.status = 'deprioritized';
+            }
           } else if (isAdded[tx.txid] && (blockAudit.version === 0 || isUnseen[tx.txid])) {
             tx.status = 'added';
           } else if (inTemplate[tx.txid]) {

frontend/src/app/interfaces/node-api.interface.ts

@@ -239,7 +239,7 @@ export interface TransactionStripped {
   acc?: boolean;
   flags?: number | null;
   time?: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'added_deprioritized' | 'deprioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
   context?: 'projected' | 'actual';
 }
 

frontend/src/app/shared/transaction.utils.ts

@@ -1,7 +1,7 @@
 import { TransactionFlags } from './filters.utils';
 import { getVarIntLength, opcodes, parseMultisigScript, isPoint } from './script.utils';
 import { Transaction } from '../interfaces/electrs.interface';
-import { CpfpInfo, RbfInfo } from '../interfaces/node-api.interface';
+import { CpfpInfo, RbfInfo, TransactionStripped } from '../interfaces/node-api.interface';
 
 // Bitcoin Core default policy settings
 const TX_MAX_STANDARD_VERSION = 2;
@@ -459,3 +459,82 @@ export function getUnacceleratedFeeRate(tx: Transaction, accelerated: boolean):
     return tx.effectiveFeePerVsize;
   }
 }
+
+export function identifyPrioritizedTransactions(transactions: TransactionStripped[]): { prioritized: string[], deprioritized: string[] } {
+  // find the longest increasing subsequence of transactions
+  // (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms)
+  // should be O(n log n)
+  const X = transactions.slice(1).reverse(); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase)
+  if (X.length < 2) {
+    return { prioritized: [], deprioritized: [] };
+  }
+  const N = X.length;
+  const P: number[] = new Array(N);
+  const M: number[] = new Array(N + 1);
+  M[0] = -1; // undefined so can be set to any value
+
+  let L = 0;
+  for (let i = 0; i < N; i++) {
+    // Binary search for the smallest positive l ≤ L
+    // such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize
+    let lo = 1;
+    let hi = L + 1;
+    while (lo < hi) {
+      const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi
+      if (X[M[mid]].rate > X[i].rate) {
+        hi = mid;
+      } else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize
+        lo = mid + 1;
+      }
+    }
+
+    // After searching, lo == hi is 1 greater than the
+    // length of the longest prefix of X[i]
+    const newL = lo;
+
+    // The predecessor of X[i] is the last index of
+    // the subsequence of length newL-1
+    P[i] = M[newL - 1];
+    M[newL] = i;
+
+    if (newL > L) {
+      // If we found a subsequence longer than any we've
+      // found yet, update L
+      L = newL;
+    }
+  }
+
+  // Reconstruct the longest increasing subsequence
+  // It consists of the values of X at the L indices:
+  // ..., P[P[M[L]]], P[M[L]], M[L]
+  const LIS: TransactionStripped[] = new Array(L);
+  let k = M[L];
+  for (let j = L - 1; j >= 0; j--) {
+    LIS[j] = X[k];
+    k = P[k];
+  }
+
+  const lisMap = new Map<string, number>();
+  LIS.forEach((tx, index) => lisMap.set(tx.txid, index));
+
+  const prioritized: string[] = [];
+  const deprioritized: string[] = [];
+
+  let lastRate = 0;
+
+  for (const tx of X) {
+    if (lisMap.has(tx.txid)) {
+      lastRate = tx.rate;
+    } else {
+      if (Math.abs(tx.rate - lastRate) < 0.1) {
+        // skip if the rate is almost the same as the previous transaction
+      } else if (tx.rate <= lastRate) {
+        prioritized.push(tx.txid);
+      } else {
+        deprioritized.push(tx.txid);
+      }
+    }
+  }
+
+  return { prioritized, deprioritized };
+}