From 82f8bf6bb489aa2c8c9715ea779c52b61a136b26 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 18:47:45 +0200 Subject: [PATCH 1/4] Refactor ISP pie chart to make it more consitent --- backend/src/api/explorer/nodes.api.ts | 147 ++++++++++++------ backend/src/api/explorer/nodes.routes.ts | 10 +- .../nodes-per-isp-chart.component.html | 45 +++--- .../nodes-per-isp-chart.component.scss | 7 +- .../nodes-per-isp-chart.component.ts | 110 +++++++------ frontend/src/app/services/api.service.ts | 5 +- .../components/toggle/toggle.component.html | 2 +- .../components/toggle/toggle.component.ts | 1 + 8 files changed, 192 insertions(+), 135 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 2d838524e..e59770d50 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -168,64 +168,115 @@ class NodesApi { } } - public async $getNodesISPRanking(groupBy: string, showTor: boolean) { + public async $getNodesISPRanking() { try { - const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`; - - // Clearnet - let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names, - COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity - FROM nodes - JOIN geo_names ON geo_names.id = nodes.as_number - JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key - GROUP BY geo_names.names - ORDER BY ${orderBy} DESC - `; - const [nodesCountPerAS]: any = await DB.query(query); + let query = ''; - let total = 0; - const nodesPerAs: any[] = []; + // List all channels and the two linked ISP + query = ` + SELECT short_id, capacity, + channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID, + channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID + FROM channels + JOIN nodes node1 ON node1.public_key = channels.node1_public_key + JOIN nodes node2 ON node2.public_key = channels.node2_public_key + JOIN geo_names isp1 ON isp1.id = node1.as_number + JOIN geo_names isp2 ON isp2.id = node2.as_number + WHERE channels.status = 1 + ORDER BY short_id DESC + `; + const [channelsIsp]: any = await DB.query(query); - for (const asGroup of nodesCountPerAS) { - if (groupBy === 'capacity') { - total += asGroup.capacity; - } else { - total += asGroup.nodesCount; + // Sum channels capacity and node count per ISP + const ispList = {}; + for (const channel of channelsIsp) { + const isp1 = JSON.parse(channel.isp1); + const isp2 = JSON.parse(channel.isp2); + + if (!ispList[isp1]) { + ispList[isp1] = { + id: channel.isp1ID, + capacity: 0, + channels: 0, + nodes: {}, + }; } + if (!ispList[isp2]) { + ispList[isp2] = { + id: channel.isp2ID, + capacity: 0, + channels: 0, + nodes: {}, + }; + } + + ispList[isp1].capacity += channel.capacity; + ispList[isp1].channels += 1; + ispList[isp1].nodes[channel.node1PublicKey] = true; + ispList[isp2].capacity += channel.capacity; + ispList[isp2].channels += 1; + ispList[isp2].nodes[channel.node2PublicKey] = true; } - // Tor - if (showTor) { - query = `SELECT COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity - FROM nodes - JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key - ORDER BY ${orderBy} DESC - `; - const [nodesCountTor]: any = await DB.query(query); - - total += groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount; - nodesPerAs.push({ - ispId: null, - name: 'Tor', - count: nodesCountTor[0].nodesCount, - share: Math.floor((groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount) / total * 10000) / 100, - capacity: nodesCountTor[0].capacity, - }); + const ispRanking: any[] = []; + for (const isp of Object.keys(ispList)) { + ispRanking.push([ + ispList[isp].id, + isp, + ispList[isp].capacity, + ispList[isp].channels, + Object.keys(ispList[isp].nodes).length, + ]); } - for (const as of nodesCountPerAS) { - nodesPerAs.push({ - ispId: as.ispId, - name: JSON.parse(as.names), - count: as.nodesCount, - share: Math.floor((groupBy === 'capacity' ? as.capacity : as.nodesCount) / total * 10000) / 100, - capacity: as.capacity, - }); - } + // Total active channels capacity + query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`; + const [totalCapacity]: any = await DB.query(query); - return nodesPerAs; + // Get the total capacity of all channels which have at least one node on clearnet + query = ` + SELECT SUM(capacity) as capacity + FROM ( + SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks + FROM channels + JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key + JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key + AND channels.status = 1 + GROUP BY short_id + ) channels_tmp + WHERE channels_tmp.networks LIKE '%ipv%' + `; + const [clearnetCapacity]: any = await DB.query(query); + + // Get the total capacity of all channels which have both nodes on Tor + query = ` + SELECT SUM(capacity) as capacity + FROM ( + SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks + FROM channels + JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key + JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key + AND channels.status = 1 + GROUP BY short_id + ) channels_tmp + WHERE channels_tmp.networks NOT LIKE '%ipv%' AND + channels_tmp.networks NOT LIKE '%dns%' AND + channels_tmp.networks NOT LIKE '%websocket%' + `; + const [torCapacity]: any = await DB.query(query); + + const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10); + const torCapacityValue = parseInt(torCapacity[0].capacity, 10); + const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue; + + return { + clearnetCapacity: clearnetCapacityValue, + torCapacity: torCapacityValue, + unknownCapacity: unknownCapacityValue, + ispRanking: ispRanking, + }; } catch (e) { - logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`); + logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`); throw e; } } diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index 5e0f95acb..a07001c8d 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -71,15 +71,7 @@ class NodesRoutes { private async $getISPRanking(req: Request, res: Response): Promise { try { - const groupBy = req.query.groupBy as string; - const showTor = req.query.showTor as string === 'true' ? true : false; - - if (!['capacity', 'node-count'].includes(groupBy)) { - res.status(400).send(`groupBy must be one of 'capacity' or 'node-count'`); - return; - } - - const nodesPerAs = await nodesApi.$getNodesISPRanking(groupBy, showTor); + const nodesPerAs = await nodesApi.$getNodesISPRanking(); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html index 01be4f036..25773a06e 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html @@ -3,21 +3,24 @@
-
Tagged ISPs
-

- {{ stats.taggedISP }} +

Clearnet capacity
+

+

-
Tagged nodes
-

- {{ stats.taggedNodeCount }} +

Unknown capacity
+

+

-
Tagged capacity
-

- +

Tor capacity
+

+

@@ -25,13 +28,13 @@
- Lightning nodes per ISP + Top 100 ISP hosting LN nodes
- (Tor nodes excluded) + (Tor nodes excluded)
@@ -44,9 +47,8 @@
-
- - +
+
@@ -59,16 +61,15 @@ - - - + + + - - - + + +
Capacity
{{ asEntry.rank }}
{{ isp[5] }} - {{ asEntry.name }} - {{ asEntry.name }} + {{ isp[1] }} {{ asEntry.count }}{{ isp[4] }}
diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss index 874d901b2..c6897cda9 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss @@ -149,7 +149,8 @@ } .name { - width: 25%; + width: 35%; + max-width: 300px; @media (max-width: 576px) { width: 70%; max-width: 150px; @@ -159,14 +160,14 @@ } .share { - width: 20%; + width: 15%; @media (max-width: 576px) { display: none } } .nodes { - width: 20%; + width: 15%; @media (max-width: 576px) { width: 10%; } diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts index cd8a72884..116c3215c 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts @@ -26,14 +26,15 @@ export class NodesPerISPChartComponent implements OnInit { renderer: 'svg', }; timespan = ''; + sortBy = 'capacity'; + showUnknown = false; chartInstance = undefined; @HostBinding('attr.dir') dir = 'ltr'; nodesPerAsObservable$: Observable; - showTorObservable$: Observable; - groupBySubject = new Subject(); - showTorSubject = new Subject(); + sortBySubject = new Subject(); + showUnknownSubject = new Subject(); constructor( private apiService: ApiService, @@ -48,32 +49,49 @@ export class NodesPerISPChartComponent implements OnInit { ngOnInit(): void { this.seoService.setTitle($localize`Lightning nodes per ISP`); - this.showTorObservable$ = this.showTorSubject.asObservable(); - this.nodesPerAsObservable$ = combineLatest([ - this.groupBySubject.pipe(startWith(false)), - this.showTorSubject.pipe(startWith(false)), + this.sortBySubject.pipe(startWith(true)), ]) .pipe( switchMap((selectedFilters) => { - return this.apiService.getNodesPerAs( - selectedFilters[0] ? 'capacity' : 'node-count', - selectedFilters[1] // Show Tor nodes - ) + this.sortBy = selectedFilters[0] ? 'capacity' : 'node-count'; + return this.apiService.getNodesPerIsp() .pipe( - tap(data => { + tap(() => { this.isLoading = false; - this.prepareChartOptions(data); }), map(data => { - for (let i = 0; i < data.length; ++i) { - data[i].rank = i + 1; + let nodeCount = 0; + let totalCapacity = 0; + + for (let i = 0; i < data.ispRanking.length; ++i) { + nodeCount += data.ispRanking[i][4]; + totalCapacity += data.ispRanking[i][2]; + data.ispRanking[i][5] = i; } + for (let i = 0; i < data.ispRanking.length; ++i) { + data.ispRanking[i][6] = Math.round(data.ispRanking[i][4] / nodeCount * 10000) / 100; + data.ispRanking[i][7] = Math.round(data.ispRanking[i][2] / totalCapacity * 10000) / 100; + } + + if (selectedFilters[0] === true) { + data.ispRanking.sort((a, b) => b[7] - a[7]); + } else { + data.ispRanking.sort((a, b) => b[6] - a[6]); + } + + for (let i = 0; i < data.ispRanking.length; ++i) { + data.ispRanking[i][5] = i + 1; + } + + this.prepareChartOptions(data.ispRanking); + return { - taggedISP: data.length, - taggedCapacity: data.reduce((partialSum, isp) => partialSum + isp.capacity, 0), - taggedNodeCount: data.reduce((partialSum, isp) => partialSum + isp.count, 0), - data: data.slice(0, 100), + taggedISP: data.ispRanking.length, + clearnetCapacity: data.clearnetCapacity, + unknownCapacity: data.unknownCapacity, + torCapacity: data.torCapacity, + ispRanking: data.ispRanking.slice(0, 100), }; }) ); @@ -82,22 +100,22 @@ export class NodesPerISPChartComponent implements OnInit { ); if (this.widget) { - this.showTorSubject.next(false); - this.groupBySubject.next(false); + this.sortBySubject.next(false); } } - generateChartSerieData(as): PieSeriesOption[] { + generateChartSerieData(ispRanking): PieSeriesOption[] { let shareThreshold = 0.5; if (this.widget && isMobile() || isMobile()) { shareThreshold = 1; } else if (this.widget) { shareThreshold = 0.75; } - + const data: object[] = []; let totalShareOther = 0; - let totalNodeOther = 0; + let nodeCountOther = 0; + let capacityOther = 0; let edgeDistance: string | number = '10%'; if (isMobile() && this.widget) { @@ -106,18 +124,19 @@ export class NodesPerISPChartComponent implements OnInit { edgeDistance = 10; } - as.forEach((as) => { - if (as.share < shareThreshold) { - totalShareOther += as.share; - totalNodeOther += as.count; + ispRanking.forEach((isp) => { + if ((this.sortBy === 'capacity' ? isp[7] : isp[6]) < shareThreshold) { + totalShareOther += this.sortBy === 'capacity' ? isp[7] : isp[6]; + nodeCountOther += isp[4]; + capacityOther += isp[2]; return; } data.push({ itemStyle: { - color: as.ispId === null ? '#7D4698' : undefined, + color: isp[0] === null ? '#7D4698' : undefined, }, - value: as.share, - name: as.name + (isMobile() || this.widget ? `` : ` (${as.share}%)`), + value: this.sortBy === 'capacity' ? isp[7] : isp[6], + name: isp[1].replace('&', '') + (isMobile() || this.widget ? `` : ` (${this.sortBy === 'capacity' ? isp[7] : isp[6]}%)`), label: { overflow: 'truncate', width: isMobile() ? 75 : this.widget ? 125 : 250, @@ -135,13 +154,13 @@ export class NodesPerISPChartComponent implements OnInit { }, borderColor: '#000', formatter: () => { - return `${as.name} (${as.share}%)
` + - $localize`${as.count.toString()} nodes
` + - $localize`${this.amountShortenerPipe.transform(as.capacity / 100000000, 2)} BTC capacity` + return `${isp[1]} (${isp[6]}%)
` + + $localize`${isp[4].toString()} nodes
` + + $localize`${this.amountShortenerPipe.transform(isp[2] / 100000000, 2)} BTC` ; } }, - data: as.ispId, + data: isp[0], } as PieSeriesOption); }); @@ -167,8 +186,9 @@ export class NodesPerISPChartComponent implements OnInit { }, borderColor: '#000', formatter: () => { - return `${'Other'} (${totalShareOther.toFixed(2)}%)
` + - totalNodeOther.toString() + ` nodes`; + return `Other (${totalShareOther.toFixed(2)}%)
` + + $localize`${nodeCountOther.toString()} nodes
` + + $localize`${this.amountShortenerPipe.transform(capacityOther / 100000000, 2)} BTC`; } }, data: 9999 as any, @@ -177,7 +197,7 @@ export class NodesPerISPChartComponent implements OnInit { return data; } - prepareChartOptions(as): void { + prepareChartOptions(ispRanking): void { let pieSize = ['20%', '80%']; // Desktop if (isMobile() && !this.widget) { pieSize = ['15%', '60%']; @@ -194,11 +214,11 @@ export class NodesPerISPChartComponent implements OnInit { series: [ { zlevel: 0, - minShowLabelAngle: 1.8, + minShowLabelAngle: 0.9, name: 'Lightning nodes', type: 'pie', radius: pieSize, - data: this.generateChartSerieData(as), + data: this.generateChartSerieData(ispRanking), labelLine: { lineStyle: { width: 2, @@ -259,16 +279,8 @@ export class NodesPerISPChartComponent implements OnInit { this.chartInstance.setOption(this.chartOptions); } - onTorToggleStatusChanged(e): void { - this.showTorSubject.next(e); - } - onGroupToggleStatusChanged(e): void { - this.groupBySubject.next(e); - } - - isEllipsisActive(e) { - return (e.offsetWidth < e.scrollWidth); + this.sortBySubject.next(e); } } diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 844451574..f7f4cb5b6 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -255,9 +255,8 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/search', { params }); } - getNodesPerAs(groupBy: 'capacity' | 'node-count', showTorNodes: boolean): Observable { - return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp-ranking' - + `?groupBy=${groupBy}&showTor=${showTorNodes}`); + getNodesPerIsp(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp-ranking'); } getNodeForCountry$(country: string): Observable { diff --git a/frontend/src/app/shared/components/toggle/toggle.component.html b/frontend/src/app/shared/components/toggle/toggle.component.html index dac33c9d8..ea67f5416 100644 --- a/frontend/src/app/shared/components/toggle/toggle.component.html +++ b/frontend/src/app/shared/components/toggle/toggle.component.html @@ -1,7 +1,7 @@
{{ textLeft }}   {{ textRight }} diff --git a/frontend/src/app/shared/components/toggle/toggle.component.ts b/frontend/src/app/shared/components/toggle/toggle.component.ts index 4bd31ffbd..f389989d9 100644 --- a/frontend/src/app/shared/components/toggle/toggle.component.ts +++ b/frontend/src/app/shared/components/toggle/toggle.component.ts @@ -10,6 +10,7 @@ export class ToggleComponent implements AfterViewInit { @Output() toggleStatusChanged = new EventEmitter(); @Input() textLeft: string; @Input() textRight: string; + @Input() checked: boolean = false; ngAfterViewInit(): void { this.toggleStatusChanged.emit(false); From 264ce1222ae0a35be897567e747e84fbe557812c Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 19:00:08 +0200 Subject: [PATCH 2/4] Remove "invalid data skipping fix" from stats importer --- .../lightning/sync-tasks/stats-importer.ts | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 7b618e66e..a75e83142 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -63,6 +63,9 @@ class LightningStatsImporter { let isUnnanounced = true; for (const socket of (node.addresses ?? [])) { + if (!socket.network?.length || !socket.addr?.length) { + continue; + } hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1; hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])); } @@ -263,8 +266,6 @@ class LightningStatsImporter { * Import topology files LN historical data into the database */ async $importHistoricalLightningStats(): Promise { - let latestNodeCount = 1; - const fileList = await fsPromises.readdir(this.topologiesFolder); // Insert history from the most recent to the oldest // This also put the .json cached files first @@ -292,12 +293,18 @@ class LightningStatsImporter { // Stats exist already, don't calculate/insert them if (existingStatsTimestamps[timestamp] !== undefined) { - latestNodeCount = existingStatsTimestamps[timestamp].node_count; continue; } logger.debug(`Reading ${this.topologiesFolder}/${filename}`); - const fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); + let fileContent = ''; + try { + fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); + } catch (e: any) { + if (e.errno == -1) { // EISDIR - Ignore directorie + continue; + } + } let graph; if (filename.indexOf('.json') !== -1) { @@ -316,18 +323,6 @@ class LightningStatsImporter { await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); } - if (timestamp > 1556316000) { - // "No, the reason most likely is just that I started collection in 2019, - // so what I had before that is just the survivors from before, which weren't that many" - const diffRatio = graph.nodes.length / latestNodeCount; - if (diffRatio < 0.9) { - // Ignore drop of more than 90% of the node count as it's probably a missing data point - logger.debug(`Nodes count diff ratio threshold reached, ignore the data for this day ${graph.nodes.length} nodes vs ${latestNodeCount}`); - continue; - } - } - latestNodeCount = graph.nodes.length; - const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); From a71262f538d1083a70f2a39cd014240e1b8e927f Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 19:29:00 +0200 Subject: [PATCH 3/4] Assume topology file are in .json - trim log --- .../lightning/sync-tasks/stats-importer.ts | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index a75e83142..f73ab4b47 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -38,8 +38,6 @@ class LightningStatsImporter { parser = new XMLParser(); async $run(): Promise { - logger.info(`Importing historical lightning stats`); - const [channels]: any[] = await DB.query('SELECT short_id from channels;'); logger.info('Caching funding txs for currently existing channels'); await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); @@ -283,11 +281,11 @@ class LightningStatsImporter { // For logging purpose let processed = 10; - let totalProcessed = -1; + let totalProcessed = 0; + let logStarted = false; for (const filename of fileList) { processed++; - totalProcessed++; const timestamp = parseInt(filename.split('_')[1], 10); @@ -296,6 +294,10 @@ class LightningStatsImporter { continue; } + if (filename.indexOf('.json') === -1) { + continue; + } + logger.debug(`Reading ${this.topologiesFolder}/${filename}`); let fileContent = ''; try { @@ -307,25 +309,23 @@ class LightningStatsImporter { } let graph; - if (filename.indexOf('.json') !== -1) { - try { - graph = JSON.parse(fileContent); - } catch (e) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); - continue; - } - } else { - graph = this.parseFile(fileContent); - if (!graph) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); - continue; - } - await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); + try { + graph = JSON.parse(fileContent); + } catch (e) { + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + continue; } - + + if (!logStarted) { + logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`); + logStarted = true; + } + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); + totalProcessed++; + if (processed > 10) { logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); processed = 0; @@ -338,7 +338,9 @@ class LightningStatsImporter { existingStatsTimestamps[timestamp] = stat; } - logger.info(`Lightning network stats historical import completed`); + if (totalProcessed > 0) { + logger.info(`Lightning network stats historical import completed`); + } } /** From 8dc41257ce9b9b1622e2fd3ed03e006c0dbd7a8b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 21:47:52 +0200 Subject: [PATCH 4/4] Remove xml parser - Read only .topology file and assume json format --- backend/package-lock.json | 34 --------- backend/package.json | 1 - .../lightning/sync-tasks/stats-importer.ts | 73 +------------------ 3 files changed, 1 insertion(+), 107 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 4e43dc309..e83ddf252 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -16,7 +16,6 @@ "bitcoinjs-lib": "6.0.2", "crypto-js": "^4.0.0", "express": "^4.18.0", - "fast-xml-parser": "^4.0.9", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", @@ -3136,21 +3135,6 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "node_modules/fast-xml-parser": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", - "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - }, - "funding": { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -5636,11 +5620,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -8556,14 +8535,6 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "fast-xml-parser": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", - "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", - "requires": { - "strnum": "^1.0.5" - } - }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -10398,11 +10369,6 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, - "strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/backend/package.json b/backend/package.json index 084b57731..082449dac 100644 --- a/backend/package.json +++ b/backend/package.json @@ -38,7 +38,6 @@ "bitcoinjs-lib": "6.0.2", "crypto-js": "^4.0.0", "express": "^4.18.0", - "fast-xml-parser": "^4.0.9", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index f73ab4b47..5878f898a 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -1,6 +1,5 @@ import DB from '../../../database'; import { promises } from 'fs'; -import { XMLParser } from 'fast-xml-parser'; import logger from '../../../logger'; import fundingTxFetcher from './funding-tx-fetcher'; import config from '../../../config'; @@ -35,7 +34,6 @@ interface Channel { class LightningStatsImporter { topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; - parser = new XMLParser(); async $run(): Promise { const [channels]: any[] = await DB.query('SELECT short_id from channels;'); @@ -294,7 +292,7 @@ class LightningStatsImporter { continue; } - if (filename.indexOf('.json') === -1) { + if (filename.indexOf('.topology') === -1) { continue; } @@ -342,75 +340,6 @@ class LightningStatsImporter { logger.info(`Lightning network stats historical import completed`); } } - - /** - * Parse the file content into XML, and return a list of nodes and channels - */ - private parseFile(fileContent): any { - const graph = this.parser.parse(fileContent); - if (Object.keys(graph).length === 0) { - return null; - } - - const nodes: Node[] = []; - const channels: Channel[] = []; - - // If there is only one entry, the parser does not return an array, so we override this - if (!Array.isArray(graph.graphml.graph.node)) { - graph.graphml.graph.node = [graph.graphml.graph.node]; - } - if (!Array.isArray(graph.graphml.graph.edge)) { - graph.graphml.graph.edge = [graph.graphml.graph.edge]; - } - - for (const node of graph.graphml.graph.node) { - if (!node.data) { - continue; - } - const addresses: unknown[] = []; - const sockets = node.data[5].split(','); - for (const socket of sockets) { - const parts = socket.split('://'); - addresses.push({ - network: parts[0], - addr: parts[1], - }); - } - nodes.push({ - id: node.data[0], - timestamp: node.data[1], - features: node.data[2], - rgb_color: node.data[3], - alias: node.data[4], - addresses: addresses, - out_degree: node.data[6], - in_degree: node.data[7], - }); - } - - for (const channel of graph.graphml.graph.edge) { - if (!channel.data) { - continue; - } - channels.push({ - channel_id: channel.data[0], - node1_pub: channel.data[1], - node2_pub: channel.data[2], - timestamp: channel.data[3], - features: channel.data[4], - fee_base_msat: channel.data[5], - fee_rate_milli_msat: channel.data[6], - htlc_minimim_msat: channel.data[7], - cltv_expiry_delta: channel.data[8], - htlc_maximum_msat: channel.data[9], - }); - } - - return { - nodes: nodes, - edges: channels, - }; - } } export default new LightningStatsImporter;