Merge pull request #402 from mempool/simon/mempool-lazy-delete

Flag transactions for lazy deletion.

backend/src/api/mempool.ts

@@ -10,6 +10,7 @@ import loadingIndicators from './loading-indicators';
 
 class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
+  private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
   private mempoolCache: { [txId: string]: TransactionExtended } = {};
   private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
@@ -27,6 +28,7 @@ class Mempool {
 
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
+    setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
   public isInSync(): boolean {
@@ -145,7 +147,6 @@ class Mempool {
       }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
     }
 
-    let newMempool = {};
     const deletedTransactions: TransactionExtended[] = [];
 
     if (this.mempoolProtection !== 1) {
@@ -154,35 +155,31 @@ class Mempool {
       const transactionsObject = {};
       transactions.forEach((txId) => transactionsObject[txId] = true);
 
-      // Replace mempool to separate deleted transactions
+      // Flag transactions for lazy deletion
       for (const tx in this.mempoolCache) {
-        if (transactionsObject[tx]) {
-          newMempool[tx] = this.mempoolCache[tx];
-        } else {
+        if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
           deletedTransactions.push(this.mempoolCache[tx]);
+          this.mempoolCache[tx].deleteAfter = new Date().getTime() + Mempool.LAZY_DELETE_AFTER_SECONDS * 1000;
         }
       }
-    } else {
-      newMempool = this.mempoolCache;
     }
 
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
-    if (!this.inSync && transactions.length === Object.keys(newMempool).length) {
+    if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
       this.inSync = true;
       logger.info('The mempool is now in sync!');
       loadingIndicators.setProgress('mempool', 100);
     }
 
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      this.mempoolCache = newMempool;
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(newMempool).length} Change: ${diff}`);
+    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
     logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
   }
 
@@ -198,6 +195,16 @@ class Mempool {
       );
     }
   }
+
+  private deleteExpiredTransactions() {
+    const now = new Date().getTime();
+    for (const tx in this.mempoolCache) {
+      const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
+      if (lazyDeleteAt && lazyDeleteAt < now) {
+        delete this.mempoolCache[tx];
+      }
+    }
+  }
 }
 
 export default new Mempool();

backend/src/api/websocket-handler.ts

@@ -194,10 +194,15 @@ class WebsocketHandler {
 
     mempoolBlocks.updateMempoolBlocks(newMempool);
     const mBlocks = mempoolBlocks.getMempoolBlocks();
+    const mempool = memPool.getMempool();
     const mempoolInfo = memPool.getMempoolInfo();
     const vBytesPerSecond = memPool.getVBytesPerSecond();
     const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
 
+    for (const rbfTransaction in rbfTransactions) {
+      delete mempool[rbfTransaction];
+    }
+
     this.wss.clients.forEach(async (client: WebSocket) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
@@ -329,28 +334,23 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    // Check how many transactions in the new block matches the latest projected mempool block
-    // If it's more than 0, recalculate the mempool blocks and send to client in the same update
     let mBlocks: undefined | MempoolBlock[];
     let matchRate = 0;
+    const _memPool = memPool.getMempool();
     const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+
     if (_mempoolBlocks[0]) {
       const matches: string[] = [];
       for (const txId of txIds) {
         if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
           matches.push(txId);
         }
+        delete _memPool[txId];
       }
 
       matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
-      if (matchRate > 0) {
-        const currentMemPool = memPool.getMempool();
-        for (const txId of matches) {
-          delete currentMemPool[txId];
-        }
-        mempoolBlocks.updateMempoolBlocks(currentMemPool);
-        mBlocks = mempoolBlocks.getMempoolBlocks();
-      }
+      mempoolBlocks.updateMempoolBlocks(_memPool);
+      mBlocks = mempoolBlocks.getMempoolBlocks();
     }
 
     block.matchRate = matchRate;

backend/src/mempool.interfaces.ts

@@ -30,6 +30,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   ancestors?: Ancestor[];
   bestDescendant?: BestDescendant | null;
   cpfpChecked?: boolean;
+  deleteAfter?: number;
 }
 
 interface Ancestor {