Merge branch 'master' into nymkappa/bugfix/missing-node-location

backend/src/tasks/lightning/network-sync.service.ts

@@ -83,7 +83,7 @@ class NetworkSyncService {
     logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
 
     // If a channel if not present in the graph, mark it as inactive
-    nodesApi.$setNodesInactive(graphNodesPubkeys);
+    await nodesApi.$setNodesInactive(graphNodesPubkeys);
 
     if (config.MAXMIND.ENABLED) {
       $lookupNodeLocation();
@@ -121,7 +121,7 @@ class NetworkSyncService {
       logger.info(`${progress} channels updated`);
 
       // If a channel if not present in the graph, mark it as inactive
-      channelsApi.$setChannelsInactive(graphChannelsIds);
+      await channelsApi.$setChannelsInactive(graphChannelsIds);
     } catch (e) {
       logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
     }

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -7,6 +7,7 @@ import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
 import { isIP } from 'net';
 import { Common } from '../../../api/common';
 import channelsApi from '../../../api/explorer/channels.api';
+import nodesApi from '../../../api/explorer/nodes.api';
 
 const fsPromises = promises;
 
@@ -32,7 +33,26 @@ class LightningStatsImporter {
     let clearnetTorNodes = 0;
     let unannouncedNodes = 0;
 
+    const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`);
+    const nodesInDb = {};
+    for (const node of nodesInDbRaw) {
+      nodesInDb[node.public_key] = node;
+    }
+
     for (const node of networkGraph.nodes) {
+      // If we don't know about this node, insert it in db
+      if (isHistorical === true && !nodesInDb[node.pub_key]) {
+        await nodesApi.$saveNode({
+          last_update: node.last_update,
+          pub_key: node.pub_key,
+          alias: node.alias,
+          addresses: node.addresses,
+          color: node.color,
+          features: node.features,
+        });
+        nodesInDb[node.pub_key] = node;
+      }
+
       let hasOnion = false;
       let hasClearnet = false;
       let isUnnanounced = true;
@@ -69,7 +89,7 @@ class LightningStatsImporter {
     const baseFees: number[] = [];
     const alreadyCountedChannels = {};
     
-    const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id, created FROM channels`);
+    const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`);
     const channelsInDb = {};
     for (const channel of channelsInDbRaw) {
       channelsInDb[channel.short_id] = channel;
@@ -84,29 +104,19 @@ class LightningStatsImporter {
         continue;
       }
 
-      // Channel is already in db, check if we need to update 'created' field
+      // If we don't know about this channel, insert it in db
-      if (isHistorical === true) {
+      if (isHistorical === true && !channelsInDb[short_id]) {
-        //@ts-ignore
+        await channelsApi.$saveChannel({
-        if (channelsInDb[short_id] && channel.timestamp < channel.created) {
+          channel_id: short_id,
-          await DB.query(`
+          chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
-            UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.short_id = ?`,
+          last_update: channel.last_update,
-            //@ts-ignore
+          node1_pub: channel.node1_pub,
-            [channel.timestamp, short_id]
+          node2_pub: channel.node2_pub,
-          );
+          capacity: (tx.value * 100000000).toString(),
-        } else if (!channelsInDb[short_id]) {
+          node1_policy: null,
-          await channelsApi.$saveChannel({
+          node2_policy: null,
-            channel_id: short_id,
+        }, 0);
-            chan_point: `${tx.txid}:${short_id.split('x')[2]}`,
+        channelsInDb[channel.channel_id] = channel;
-            //@ts-ignore
-            last_update: channel.timestamp,
-            node1_pub: channel.node1_pub,
-            node2_pub: channel.node2_pub,
-            capacity: (tx.value * 100000000).toString(),
-            node1_policy: null,
-            node2_policy: null,
-          }, 0);
-          channelsInDb[channel.channel_id] = channel;
-        }
       }
 
       if (!nodeStats[channel.node1_pub]) {
@@ -281,6 +291,7 @@ class LightningStatsImporter {
    * Import topology files LN historical data into the database
    */
   async $importHistoricalLightningStats(): Promise<void> {
+    logger.debug('Run the historical importer');
     try {
       let fileList: string[] = [];
       try {
@@ -294,7 +305,7 @@ class LightningStatsImporter {
       fileList.sort().reverse();
 
       const [rows]: any[] = await DB.query(`
-        SELECT UNIX_TIMESTAMP(added) AS added, node_count
+        SELECT UNIX_TIMESTAMP(added) AS added
         FROM lightning_stats
         ORDER BY added DESC
       `);
@@ -391,12 +402,16 @@ class LightningStatsImporter {
         });
       }
 
+      let rgb = node.rgb_color ?? '#000000';
+      if (rgb.indexOf('#') === -1) {
+        rgb = `#${rgb}`;
+      }
       newGraph.nodes.push({
         last_update: node.timestamp ?? 0,
         pub_key: node.id ?? null,
-        alias: node.alias ?? null,
+        alias: node.alias ?? node.id.slice(0, 20),
         addresses: addresses,
-        color: node.rgb_color ?? null,
+        color: rgb,
         features: {},
       });
     }

frontend/src/app/lightning/node/node.component.html

@@ -121,7 +121,7 @@
   <div *ngIf="!error">
     <div class="row" *ngIf="node.as_number">
       <div class="col-sm">
-        <app-nodes-channels-map [style]="'nodepage'" [publicKey]="node.public_key"></app-nodes-channels-map>
+        <app-nodes-channels-map [style]="'nodepage'" [publicKey]="node.public_key" [hasLocation]="!!node.as_number"></app-nodes-channels-map>
       </div>
       <div class="col-sm">
         <app-node-statistics-chart [publicKey]="node.public_key"></app-node-statistics-chart>
@@ -135,7 +135,14 @@
     <app-node-channels style="display:block;margin-bottom: 40px" [publicKey]="node.public_key"></app-node-channels>
 
     <div class="d-flex justify-content-between">
-      <h2>Channels ({{ channelsListStatus === 'open' ? node.opened_channel_count : node.closed_channel_count }})</h2>
+      <h2 *ngIf="channelsListStatus === 'open'">
+        <span i18n="lightning.open-channels">Open channels</span>
+        <span> ({{ node.opened_channel_count }})</span>
+      </h2>
+      <h2 *ngIf="channelsListStatus === 'closed'">
+        <span i18n="lightning.open-channels">Closed channels</span>
+        <span> ({{ node.closed_channel_count }})</span>
+      </h2>
     </div>
 
     <app-channels-list [publicKey]="node.public_key"

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html

@@ -1,5 +1,5 @@
-<div [style]="style === 'widget' ? 'height: 250px' : ''">
+<div class="map-wrapper" [class]="style">
-  <div *ngIf="channelsObservable | async">
+  <ng-container *ngIf="channelsObservable | async">
     <div *ngIf="chartOptions" [class]="'full-container ' + style + (fitContainer ? ' fit-container' : '')">
       <div *ngIf="style === 'graph'" class="card-header">
         <div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px">
@@ -8,14 +8,15 @@
         <small style="color: #ffffff66" i18n="lightning.tor-nodes-excluded">(Tor nodes excluded)</small>
       </div>
 
-    <div class="chart" [class]="style" echarts [initOpts]="chartInitOptions" [options]="chartOptions" (chartInit)="onChartInit($event)"
+      <div class="chart" [class]="style" echarts [initOpts]="chartInitOptions" [options]="chartOptions"
-      (chartFinished)="onChartFinished($event)">
+        (chartInit)="onChartInit($event)" (chartFinished)="onChartFinished($event)">
+      </div>
+
+      <div *ngIf="!chartOptions && style === 'nodepage'" style="padding-top: 30px"></div>
     </div>
 
-    <div *ngIf="!chartOptions && style === 'nodepage'" style="padding-top: 30px"></div>
+    <div class="text-center loading-spinner" [class]="style" *ngIf="isLoading">
-  </div>
+      <div class="spinner-border text-light"></div>
-
+    </div>
-  <div class="text-center loading-spinner" [class]="style" *ngIf="isLoading">
+  </ng-container>
-    <div class="spinner-border text-light"></div>
+</div>
-  </div>
-</div>

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss

@@ -1,3 +1,15 @@
+.map-wrapper {
+  height: 100%;
+
+  &.widget {
+    height: 250px;
+  }
+
+  &.graph {
+    height: auto;
+  }
+}
+
 .card-header {
   border-bottom: 0;
   font-size: 18px;
@@ -93,6 +105,9 @@
   top: 50%;
   left: calc(50% - 15px);
   z-index: 100;
+  @media (max-width: 767.98px) {
+    top: 550px;
+  }
 }
 .loading-spinner.widget {
   position: absolute;
@@ -103,4 +118,22 @@
   @media (max-width: 767.98px) {
     top: 250px;
   }
-}
+}
+.loading-spinner.nodepage {
+  position: absolute;
+  top: 200px;
+  z-index: 100;
+  width: 100%;
+  left: 0;
+}
+
+.loading-spinner.channelpage {
+  position: absolute;
+  top: 400px;
+  z-index: 100;
+  width: 100%;
+  left: 0;
+  @media (max-width: 767.98px) {
+    top: 450px;
+  }
+}

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -21,6 +21,7 @@ export class NodesChannelsMap implements OnInit {
   @Input() publicKey: string | undefined;
   @Input() channel: any[] = [];
   @Input() fitContainer = false;
+  @Input() hasLocation = true;
   @Output() readyEvent = new EventEmitter();
 
   channelsObservable: Observable<any>; 
@@ -32,7 +33,7 @@ export class NodesChannelsMap implements OnInit {
   channelColor = '#466d9d';
   channelCurve = 0;
   nodeSize = 4;
-  isLoading = true;
+  isLoading = false;
 
   chartInstance = undefined;
   chartOptions: EChartsOption = {};
@@ -73,6 +74,11 @@ export class NodesChannelsMap implements OnInit {
     this.channelsObservable = this.activatedRoute.paramMap
      .pipe(
        switchMap((params: ParamMap) => {
+        this.isLoading = true;
+        if (this.style === 'channelpage' && this.channel.length === 0 || !this.hasLocation) {
+          this.isLoading = false;
+        }
+            
         return zip(
           this.assetsService.getWorldMapJson$,
           this.style !== 'channelpage' ? this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined, this.style) : [''],

frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts

@@ -121,7 +121,7 @@ export class NodesNetworksChartComponent implements OnInit {
         left: 'center',
         top: 'center',
       };
-    } else if (data.tor_nodes.length > 0) {
+    } else if (this.widget && data.tor_nodes.length > 0) {
       title = {
         textStyle: {
           color: 'grey',

frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts

@@ -113,7 +113,7 @@ export class LightningStatisticsChartComponent implements OnInit {
         left: 'center',
         top: 'center'
       };
-    } else if (data.channel_count.length > 0) {
+    } else if (this.widget && data.channel_count.length > 0) {
       title = {
         textStyle: {
           color: 'grey',

production/install

@@ -385,7 +385,7 @@ DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev liba
 
 # packages needed for mempool ecosystem
 FREEBSD_PKG=()
-FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim)
+FREEBSD_PKG+=(zsh sudo git git-lfs screen curl wget calc neovim)
 FREEBSD_PKG+=(openssh-portable py39-pip rust llvm90 jq base64 libzmq4)
 FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf)
 FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase)
@@ -1267,7 +1267,7 @@ case $OS in
         osGroupCreate "${CLN_GROUP}"
         osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}"
         osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}"
-        osSudo "${CLN_USER}" touch "${CLN_HOME}/.zshrc"
+        echo "export PATH=$PATH:$HOME/.local/bin" >> "${CLN_HOME}/.zshrc"
         osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}"
 
         echo "[*] Installing Core Lightning package"

production/syslog.conf

@@ -1,4 +1,4 @@
-local7.>=notice |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops alerts
+local7.>=err |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops alerts
 local7.>=info |/usr/local/bin/sudo -u mempool /usr/local/bin/mempool-logger mempool.ops node100
 local7.>=info /var/log/mempool
 local7.* /var/log/mempool.debug