From 584ef87fc80ce6e9678e507e70c03fec67bb8ccf Mon Sep 17 00:00:00 2001
From: softsimon <softsimon@protonmail.com>
Date: Sun, 14 Feb 2021 19:50:31 +0700
Subject: [PATCH] Optimized cache chunks. Default cache files to /cache
 directory. fixes #341

---
 backend/.gitignore                 | 11 --------
 backend/cache/.gitignore           |  1 +
 backend/mempool-config.sample.json |  2 +-
 backend/src/api/disk-cache.ts      | 42 +++++++++++++++++++++---------
 backend/src/config.ts              |  2 +-
 5 files changed, 33 insertions(+), 25 deletions(-)
 create mode 100644 backend/cache/.gitignore

diff --git a/backend/.gitignore b/backend/.gitignore
index 32c205746..c4339712c 100644
--- a/backend/.gitignore
+++ b/backend/.gitignore
@@ -41,14 +41,3 @@ testem.log
 #System Files
 .DS_Store
 Thumbs.db
-
-cache.json
-cache1.json
-cache2.json
-cache3.json
-cache4.json
-cache5.json
-cache6.json
-cache7.json
-cache8.json
-cache9.json
diff --git a/backend/cache/.gitignore b/backend/cache/.gitignore
new file mode 100644
index 000000000..a6c57f5fb
--- /dev/null
+++ b/backend/cache/.gitignore
@@ -0,0 +1 @@
+*.json
diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json
index e53df38f2..1a6a52424 100644
--- a/backend/mempool-config.sample.json
+++ b/backend/mempool-config.sample.json
@@ -6,7 +6,7 @@
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
     "POLL_RATE_MS": 2000,
-    "CACHE_DIR": "./"
+    "CACHE_DIR": "./cache/"
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts
index 2956216c8..55ff02014 100644
--- a/backend/src/api/disk-cache.ts
+++ b/backend/src/api/disk-cache.ts
@@ -5,11 +5,13 @@ import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
 import config from '../config';
+import { TransactionExtended } from '../mempool.interfaces';
 
 class DiskCache {
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + 'cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + 'cache{number}.json';
-  private static CHUNK_SIZE = 10000;
+  private static CHUNK_FILES = 25;
+
   constructor() { }
 
   async $saveCacheToDisk(): Promise<void> {
@@ -18,19 +20,24 @@ class DiskCache {
     }
     try {
       logger.debug('Writing mempool and blocks data to disk cache (async)...');
-      const mempoolChunk_1 = Object.fromEntries(Object.entries(memPool.getMempool()).slice(0, DiskCache.CHUNK_SIZE));
+
+      const mempool = memPool.getMempool();
+      const mempoolArray: TransactionExtended[] = [];
+      for (const tx in mempool) {
+        mempoolArray.push(mempool[tx]);
+      }
+
+      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
+
       await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
         blocks: blocks.getBlocks(),
-        mempool: mempoolChunk_1
+        mempool: {},
+        mempoolArray: mempoolArray.splice(0, chunkSize),
       }), {flag: 'w'});
-      for (let i = 1; i < 10; i++) {
-        const mempoolChunk = Object.fromEntries(
-          Object.entries(memPool.getMempool()).slice(
-            DiskCache.CHUNK_SIZE * i, i === 9 ? undefined : DiskCache.CHUNK_SIZE * i + DiskCache.CHUNK_SIZE
-          )
-        );
+      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
         await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
-          mempool: mempoolChunk
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
         }), {flag: 'w'});
       }
       logger.debug('Mempool and blocks data saved to disk cache');
@@ -49,13 +56,24 @@ class DiskCache {
       if (cacheData) {
         logger.info('Restoring mempool and blocks data from disk cache');
         data = JSON.parse(cacheData);
+        if (data.mempoolArray) {
+          for (const tx of data.mempoolArray) {
+            data.mempool[tx.txid] = tx;
+          }
+        }
       }
 
-      for (let i = 1; i < 10; i++) {
+      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
         const fileName = DiskCache.FILE_NAMES.replace('{number}', i.toString());
         if (fs.existsSync(fileName)) {
           const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
-          Object.assign(data.mempool, cacheData2.mempool);
+          if (cacheData2.mempoolArray) {
+            for (const tx of cacheData2.mempoolArray) {
+              data.mempool[tx.txid] = tx;
+            }
+          } else {
+            Object.assign(data.mempool, cacheData2.mempool);
+          }
         }
       }
 
diff --git a/backend/src/config.ts b/backend/src/config.ts
index 1d46ccfcb..f90bb4c8a 100644
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@@ -61,7 +61,7 @@ const defaults: IConfig = {
     'SPAWN_CLUSTER_PROCS': 0,
     'API_URL_PREFIX': '/api/v1/',
     'POLL_RATE_MS': 2000,
-    'CACHE_DIR': './'
+    'CACHE_DIR': './cache/'
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',