From 9000b6b18eed39119c75840e7054075b075e2385 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 6 Jul 2022 14:56:10 +0200 Subject: [PATCH] Index daily channel stats and show in dashboard widget --- backend/src/api/database-migration.ts | 13 +- backend/src/api/explorer/channels.api.ts | 63 +++++++++ backend/src/api/explorer/statistics.api.ts | 2 +- .../tasks/lightning/stats-updater.service.ts | 37 +++-- .../app/components/change/change.component.ts | 6 +- .../channels-statistics.component.html | 126 ++++++++++++++++++ .../channels-statistics.component.scss | 101 ++++++++++++++ .../channels-statistics.component.ts | 22 +++ .../app/lightning/lightning-api.service.ts | 4 + .../lightning-dashboard.component.html | 10 ++ .../lightning-dashboard.component.ts | 2 +- .../src/app/lightning/lightning.module.ts | 2 + .../node-statistics.component.html | 24 ++-- 13 files changed, 386 insertions(+), 26 deletions(-) create mode 100644 frontend/src/app/lightning/channels-statistics/channels-statistics.component.html create mode 100644 frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss create mode 100644 frontend/src/app/lightning/channels-statistics/channels-statistics.component.ts diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 9aa96a25e..363721c45 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 26; + private static currentVersion = 27; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -174,7 +174,7 @@ class DatabaseMigration { this.uniqueLog(logger.notice, this.blocksTruncatedMessage); await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index await this.$executeQuery(`ALTER TABLE blocks - ADD avg_fee INT UNSIGNED NULL, + ADD med_fee INT UNSIGNED NULL, ADD avg_fee_rate INT UNSIGNED NULL `); await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); @@ -265,6 +265,15 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"'); } + if (databaseSchemaVersion < 27 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); + } + } catch (e) { throw e; } diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 67d2d38e0..a1cd6a41e 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -71,6 +71,69 @@ class ChannelsApi { } } + public async $getChannelsStats(): Promise { + try { + // Feedback from zerofeerouting: + // "I would argue > 5000ppm can be ignored. Channels charging more than .5% fee are ignored by CLN for example." + const ignoredFeeRateThreshold = 5000; + const ignoredBaseFeeThreshold = 5000; + + // Capacity + let query = `SELECT AVG(capacity) AS avgCapacity FROM channels WHERE status = 1 ORDER BY capacity`; + const [avgCapacity]: any = await DB.query(query); + + query = `SELECT capacity FROM channels WHERE status = 1 ORDER BY capacity`; + let [capacity]: any = await DB.query(query); + capacity = capacity.map(capacity => capacity.capacity); + const medianCapacity = capacity[Math.floor(capacity.length / 2)]; + + // Fee rates + query = `SELECT node1_fee_rate FROM channels WHERE node1_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`; + let [feeRates1]: any = await DB.query(query); + feeRates1 = feeRates1.map(rate => rate.node1_fee_rate); + query = `SELECT node2_fee_rate FROM channels WHERE node2_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`; + let [feeRates2]: any = await DB.query(query); + feeRates2 = feeRates2.map(rate => rate.node2_fee_rate); + + let feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b); + let avgFeeRate = 0; + for (const rate of feeRates) { + avgFeeRate += rate; + } + avgFeeRate /= feeRates.length; + const medianFeeRate = feeRates[Math.floor(feeRates.length / 2)]; + + // Base fees + query = `SELECT node1_base_fee_mtokens FROM channels WHERE node1_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`; + let [baseFees1]: any = await DB.query(query); + baseFees1 = baseFees1.map(rate => rate.node1_base_fee_mtokens); + query = `SELECT node2_base_fee_mtokens FROM channels WHERE node2_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`; + let [baseFees2]: any = await DB.query(query); + baseFees2 = baseFees2.map(rate => rate.node2_base_fee_mtokens); + + let baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b); + let avgBaseFee = 0; + for (const fee of baseFees) { + avgBaseFee += fee; + } + avgBaseFee /= baseFees.length; + const medianBaseFee = feeRates[Math.floor(baseFees.length / 2)]; + + return { + avgCapacity: parseInt(avgCapacity[0].avgCapacity, 10), + avgFeeRate: avgFeeRate, + avgBaseFee: avgBaseFee, + medianCapacity: medianCapacity, + medianFeeRate: medianFeeRate, + medianBaseFee: medianBaseFee, + } + + } catch (e) { + logger.err(`Cannot calculate channels statistics. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + public async $getChannelsByTransactionId(transactionIds: string[]): Promise { try { transactionIds = transactionIds.map((id) => '\'' + id + '\''); diff --git a/backend/src/api/explorer/statistics.api.ts b/backend/src/api/explorer/statistics.api.ts index 3a9451b4c..71d71276b 100644 --- a/backend/src/api/explorer/statistics.api.ts +++ b/backend/src/api/explorer/statistics.api.ts @@ -16,7 +16,7 @@ class StatisticsApi { public async $getLatestStatistics(): Promise { try { const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1`); - const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1 OFFSET 72`); + const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1 OFFSET 7`); return { latest: rows[0], previous: rows2[0], diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 85c705936..3532293c6 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -2,6 +2,7 @@ import DB from '../../database'; import logger from '../../logger'; import lightningApi from '../../api/lightning/lightning-api-factory'; +import channelsApi from '../../api/explorer/channels.api'; import * as net from 'net'; class LightningStatsUpdater { @@ -124,15 +125,15 @@ class LightningStatsUpdater { ) VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`; - await DB.query(query, [ - date.getTime() / 1000, - channelsCount, - 0, - totalCapacity, - 0, - 0, - 0 - ]); + await DB.query(query, [ + date.getTime() / 1000, + channelsCount, + 0, + totalCapacity, + 0, + 0, + 0 + ]); // Add one day and continue date.setDate(date.getDate() + 1); @@ -232,6 +233,8 @@ class LightningStatsUpdater { } } + const channelStats = await channelsApi.$getChannelsStats(); + const query = `INSERT INTO lightning_stats( added, channel_count, @@ -239,7 +242,13 @@ class LightningStatsUpdater { total_capacity, tor_nodes, clearnet_nodes, - unannounced_nodes + unannounced_nodes, + avg_capacity, + avg_fee_rate, + avg_base_fee_mtokens, + med_capacity, + med_fee_rate, + med_base_fee_mtokens ) VALUES (NOW(), ?, ?, ?, ?, ?, ?)`; @@ -249,7 +258,13 @@ class LightningStatsUpdater { total_capacity, torNodes, clearnetNodes, - unannouncedNodes + unannouncedNodes, + channelStats.avgCapacity, + channelStats.avgFeeRate, + channelStats.avgBaseFee, + channelStats.medianCapacity, + channelStats.medianFeeRate, + channelStats.medianBaseFee, ]); logger.info(`Lightning daily stats done.`); } catch (e) { diff --git a/frontend/src/app/components/change/change.component.ts b/frontend/src/app/components/change/change.component.ts index 1fba853c9..c4ae965ce 100644 --- a/frontend/src/app/components/change/change.component.ts +++ b/frontend/src/app/components/change/change.component.ts @@ -15,7 +15,11 @@ export class ChangeComponent implements OnChanges { constructor() { } ngOnChanges(): void { - this.change = (this.current - this.previous) / this.previous * 100; + if (!this.previous) { + this.change = 0; + } else { + this.change = (this.current - this.previous) / this.previous * 100; + } } } diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html new file mode 100644 index 000000000..033438cf3 --- /dev/null +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.html @@ -0,0 +1,126 @@ +
+ avg + | + med +
+ +
+ +
+
+
Avg Capacity
+
+
+ {{ statistics.latest?.avg_capacity || 0 | number: '1.0-0' }} + sats +
+ + + +
+
+ +
+
Avg Fee Rate
+
+
+ {{ statistics.latest?.avg_fee_rate || 0 | number: '1.0-0' }} + ppm +
+ + + +
+
+ +
+
Avg Base Fee
+
+
+
+ {{ statistics.latest?.avg_base_fee_mtokens || 0 | number: '1.0-0' }} + msats +
+ + + +
+
+
+
+ +
+
+
Med Capacity
+
+
+ {{ statistics.latest?.med_capacity || 0 | number: '1.0-0' }} + sats +
+ + + +
+
+
+
Med Fee Rate
+
+
+ {{ statistics.latest?.med_fee_rate || 0 | number: '1.0-0' }} + ppm +
+ + + +
+
+
+
Med Base Fee
+
+
+
+ {{ statistics.latest?.med_base_fee_mtokens || 0 | number: '1.0-0' }} + msats +
+
+ + + +
+
+
+
+ + +
+
+
Nodes
+
+
+
+
+
+
+
Channels
+
+
+
+
+
+
+
Average Channel
+
+
+
+
+
+
+ \ No newline at end of file diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss new file mode 100644 index 000000000..372d9eb78 --- /dev/null +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss @@ -0,0 +1,101 @@ +.card-title { + color: #4a68b9; + font-size: 10px; + margin-bottom: 4px; + font-size: 1rem; +} + +.card-text { + font-size: 22px; + span { + font-size: 11px; + position: relative; + top: -2px; + display: inline-flex; + } + .green-color { + display: block; + } +} + +.fee-estimation-container { + display: flex; + justify-content: space-between; + @media (min-width: 376px) { + flex-direction: row; + } + .item { + max-width: 150px; + margin: 0; + width: -webkit-fill-available; + @media (min-width: 376px) { + margin: 0 auto 0px; + } + &:first-child{ + display: none; + @media (min-width: 485px) { + display: block; + } + @media (min-width: 768px) { + display: none; + } + @media (min-width: 992px) { + display: block; + } + } + &:last-child { + margin-bottom: 0; + } + .card-text span { + color: #ffffff66; + font-size: 12px; + top: 0px; + } + .fee-text{ + border-bottom: 1px solid #ffffff1c; + width: fit-content; + margin: auto; + line-height: 1.45; + padding: 0px 2px; + } + .fiat { + display: block; + font-size: 14px !important; + } + } +} + +.loading-container { + min-height: 76px; +} + +.card-text { + .skeleton-loader { + width: 100%; + display: block; + &:first-child { + max-width: 90px; + margin: 15px auto 3px; + } + &:last-child { + margin: 10px auto 3px; + max-width: 55px; + } + } +} + +.widget-toggler { + font-size: 12px; + position: absolute; + top: -20px; + right: 3px; + text-align: right; +} + +.toggler-option { + text-decoration: none; +} + +.inactive { + color: #ffffff66; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/channels-statistics/channels-statistics.component.ts b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.ts new file mode 100644 index 000000000..9fc761983 --- /dev/null +++ b/frontend/src/app/lightning/channels-statistics/channels-statistics.component.ts @@ -0,0 +1,22 @@ +import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; +import { Observable } from 'rxjs'; + +@Component({ + selector: 'app-channels-statistics', + templateUrl: './channels-statistics.component.html', + styleUrls: ['./channels-statistics.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ChannelsStatisticsComponent implements OnInit { + @Input() statistics$: Observable; + mode: string = 'avg'; + + constructor() { } + + ngOnInit(): void { + } + + switchMode(mode: 'avg' | 'med') { + this.mode = mode; + } +} diff --git a/frontend/src/app/lightning/lightning-api.service.ts b/frontend/src/app/lightning/lightning-api.service.ts index 7157c9bd7..7eeec9441 100644 --- a/frontend/src/app/lightning/lightning-api.service.ts +++ b/frontend/src/app/lightning/lightning-api.service.ts @@ -52,6 +52,10 @@ export class LightningApiService { return this.httpClient.get(this.apiBasePath + '/api/v1/lightning/nodes/top'); } + listChannelStats$(publicKey: string): Observable { + return this.httpClient.get(this.apiBasePath + '/channels/' + publicKey + '/statistics'); + } + listStatistics$(): Observable { return this.httpClient.get(this.apiBasePath + '/api/v1/lightning/statistics'); } diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html index eb840efe6..9ca9d54b8 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html @@ -19,6 +19,16 @@
Channels Statistics 
+
+
+
+ +
+
+
+ + +
diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts index 634a16cc7..5d4685fb8 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts @@ -35,7 +35,7 @@ export class LightningDashboardComponent implements OnInit { map((object) => object.topByChannels), ); - this.statistics$ = this.lightningApiService.getLatestStatistics$(); + this.statistics$ = this.lightningApiService.getLatestStatistics$().pipe(share()); } } diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index 0549fe45c..9146975e4 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -17,6 +17,7 @@ import { LightningStatisticsChartComponent } from './statistics-chart/lightning- import { NodeStatisticsChartComponent } from './node-statistics-chart/node-statistics-chart.component'; import { GraphsModule } from '../graphs/graphs.module'; import { NodesNetworksChartComponent } from './nodes-networks-chart/nodes-networks-chart.component'; +import { ChannelsStatisticsComponent } from './channels-statistics/channels-statistics.component'; @NgModule({ declarations: [ LightningDashboardComponent, @@ -31,6 +32,7 @@ import { NodesNetworksChartComponent } from './nodes-networks-chart/nodes-networ ClosingTypeComponent, LightningStatisticsChartComponent, NodesNetworksChartComponent, + ChannelsStatisticsComponent, ], imports: [ CommonModule, diff --git a/frontend/src/app/lightning/node-statistics/node-statistics.component.html b/frontend/src/app/lightning/node-statistics/node-statistics.component.html index f45daa7fd..55f68a8c3 100644 --- a/frontend/src/app/lightning/node-statistics/node-statistics.component.html +++ b/frontend/src/app/lightning/node-statistics/node-statistics.component.html @@ -2,18 +2,21 @@
Capacity
-
- +
+
+ +
- + +
Nodes
-
+
{{ statistics.latest?.node_count || 0 | number }}
@@ -24,13 +27,14 @@
Channels
-
+
{{ statistics.latest?.channel_count || 0 | number }}
- + +
@@ -73,4 +77,4 @@
- + \ No newline at end of file