mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-02-22 14:22:44 +01:00
Code Issues Projects Releases Wiki Activity

Assume topology file are in .json - trim log

Browse source
This commit is contained in:
nymkappa 2022-08-16 19:29:00 +02:00
parent 264ce1222a
commit a71262f538
No known key found for this signature in database
GPG key ID: E155910B16E8BD04
1 changed files with 22 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View file

@ -38,8 +38,6 @@ class LightningStatsImporter {
parser = new XMLParser();
async $run(): Promise<void> {
logger.info(`Importing historical lightning stats`);
const [channels]: any[] = await DB.query('SELECT short_id from channels;');
logger.info('Caching funding txs for currently existing channels');
await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
@ -283,11 +281,11 @@ class LightningStatsImporter {
// For logging purpose
let processed = 10;
let totalProcessed = -1;
let totalProcessed = 0;
let logStarted = false;
for (const filename of fileList) {
processed++;
totalProcessed++;
const timestamp = parseInt(filename.split('_')[1], 10);
@ -296,6 +294,10 @@ class LightningStatsImporter {
continue;
}
if (filename.indexOf('.json') === -1) {
continue;
}
logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
let fileContent = '';
try {
@ -307,25 +309,23 @@ class LightningStatsImporter {
}
let graph;
if (filename.indexOf('.json') !== -1) {
try {
graph = JSON.parse(fileContent);
} catch (e) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
continue;
}
} else {
graph = this.parseFile(fileContent);
if (!graph) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
continue;
}
await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph));
try {
graph = JSON.parse(fileContent);
} catch (e) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
continue;
}
if (!logStarted) {
logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
logStarted = true;
}
const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
totalProcessed++;
if (processed > 10) {
logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
processed = 0;
@ -338,7 +338,9 @@ class LightningStatsImporter {
existingStatsTimestamps[timestamp] = stat;
}
logger.info(`Lightning network stats historical import completed`);
if (totalProcessed > 0) {
logger.info(`Lightning network stats historical import completed`);
}
}
/**