mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-11-19 09:52:14 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #342 from mempool/simon/optimize-disk-cache

Browse Source 
Optimized cache chunks. Default cache files to cache/ directory.
This commit is contained in:
wiz 2021-02-14 22:11:33 +09:00 committed by GitHub
commit 2641ae0c8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 35 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/.gitignore vendored
View File

@ -41,14 +41,3 @@ testem.log
#System Files
.DS_Store
Thumbs.db
cache.json
cache1.json
cache2.json
cache3.json
cache4.json
cache5.json
cache6.json
cache7.json
cache8.json
cache9.json

1
backend/cache/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*.json

2
backend/mempool-config.sample.json
View File

@ -6,7 +6,7 @@
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"POLL_RATE_MS": 2000,
"CACHE_DIR": "./"
"CACHE_DIR": "./cache"
},
"CORE_RPC": {
"HOST": "127.0.0.1",

46
backend/src/api/disk-cache.ts
View File

@ -5,11 +5,13 @@ import memPool from './mempool';
import blocks from './blocks';
import logger from '../logger';
import config from '../config';
import { TransactionExtended } from '../mempool.interfaces';
class DiskCache {
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + 'cache.json';
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + 'cache{number}.json';
private static CHUNK_SIZE = 10000;
private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
private static CHUNK_FILES = 25;
constructor() { }
async $saveCacheToDisk(): Promise<void> {
@ -18,19 +20,24 @@ class DiskCache {
}
try {
logger.debug('Writing mempool and blocks data to disk cache (async)...');
const mempoolChunk_1 = Object.fromEntries(Object.entries(memPool.getMempool()).slice(0, DiskCache.CHUNK_SIZE));
const mempool = memPool.getMempool();
const mempoolArray: TransactionExtended[] = [];
for (const tx in mempool) {
mempoolArray.push(mempool[tx]);
}
const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
blocks: blocks.getBlocks(),
mempool: mempoolChunk_1
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), {flag: 'w'});
for (let i = 1; i < 10; i++) {
const mempoolChunk = Object.fromEntries(
Object.entries(memPool.getMempool()).slice(
DiskCache.CHUNK_SIZE * i, i === 9 ? undefined : DiskCache.CHUNK_SIZE * i + DiskCache.CHUNK_SIZE
)
);
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: mempoolChunk
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), {flag: 'w'});
}
logger.debug('Mempool and blocks data saved to disk cache');
@ -49,13 +56,24 @@ class DiskCache {
if (cacheData) {
logger.info('Restoring mempool and blocks data from disk cache');
data = JSON.parse(cacheData);
if (data.mempoolArray) {
for (const tx of data.mempoolArray) {
data.mempool[tx.txid] = tx;
}
}
}
for (let i = 1; i < 10; i++) {
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
const fileName = DiskCache.FILE_NAMES.replace('{number}', i.toString());
if (fs.existsSync(fileName)) {
const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
Object.assign(data.mempool, cacheData2.mempool);
if (cacheData2.mempoolArray) {
for (const tx of cacheData2.mempoolArray) {
data.mempool[tx.txid] = tx;
}
} else {
Object.assign(data.mempool, cacheData2.mempool);
}
}
}

2
backend/src/config.ts
View File

@ -61,7 +61,7 @@ const defaults: IConfig = {
'SPAWN_CLUSTER_PROCS': 0,
'API_URL_PREFIX': '/api/v1/',
'POLL_RATE_MS': 2000,
'CACHE_DIR': './'
'CACHE_DIR': './cache'
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',