Merge branch 'master' into nymkappa/feature/show-more-mining-pool-pie

backend/src/api/common.ts

@@ -1,5 +1,7 @@
 import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
+import { NodeSocket } from '../repositories/NodesSocketsRepository';
+import { isIP } from 'net';
 export class Common {
   static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
     '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -221,4 +223,35 @@ export class Common {
     const d = new Date((date || 0) * 1000);
     return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
   }
+
+  static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
+    let network: string | null = null;
+
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      network = socket.network;
+    } else if (config.LIGHTNING.BACKEND === 'lnd') {
+      if (socket.addr.indexOf('onion') !== -1) {
+        if (socket.addr.split('.')[0].length >= 56) {
+          network = 'torv3';
+        } else {
+          network = 'torv2';
+        }
+      } else if (socket.addr.indexOf('i2p') !== -1) {
+        network = 'i2p';
+      } else {
+        const ipv = isIP(socket.addr.split(':')[0]);
+        if (ipv === 4) {
+          network = 'ipv4';
+        } else if (ipv === 6) {
+          network = 'ipv6';
+        }
+      }
+    }
+
+    return {
+      publicKey: publicKey,
+      network: network,
+      addr: socket.addr,
+    };
+  }
 }

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 36;
+  private static currentVersion = 37;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -324,6 +324,10 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 36 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
     }
+
+    if (databaseSchemaVersion < 37 && isBitcoin == true) {
+      await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+    }
   }
 
   /**
@@ -737,7 +741,7 @@ class DatabaseMigration {
       names text DEFAULT NULL,
       UNIQUE KEY id (id,type),
       KEY id_2 (id)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
   private getCreateBlocksPricesTableQuery(): string {
@@ -749,6 +753,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodesSocketsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_sockets (
+      public_key varchar(66) NOT NULL,
+      socket varchar(100) NOT NULL,
+      type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL,
+      UNIQUE KEY public_key_socket (public_key, socket),
+      INDEX (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -17,7 +17,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
         network: addr.type,
         addr: `${addr.address}:${addr.port}`
       };
-    }),
+    }) ?? [],
     last_update: clNode?.last_timestamp ?? 0,
   };
 }

backend/src/api/lightning/lightning-api.interface.ts

@@ -82,4 +82,4 @@ export namespace ILightningApi {
     is_required: boolean;
     is_known: boolean;
   }
-}
+}

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,4 +1,3 @@
-import transactionUtils from '../api/transaction-utils';
 import DB from '../database';
 import logger from '../logger';
 import { BlockAudit } from '../mempool.interfaces';

backend/src/repositories/NodesSocketsRepository.ts

@@ -0,0 +1,45 @@
+import { ResultSetHeader } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeSocket {
+  publicKey: string;
+  network: string | null;
+  addr: string;
+}
+
+class NodesSocketsRepository {
+  public async $saveSocket(socket: NodeSocket): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO nodes_sockets(public_key, socket, type)
+        VALUE (?, ?, ?)
+      `, [socket.publicKey, socket.addr, socket.network]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes sockets
+      }
+    }
+   }
+
+   public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
+    if (addresses.length === 0) {
+      return 0;
+    }
+    try {
+      const query = `
+        DELETE FROM nodes_sockets
+        WHERE public_key = ?
+        AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
+      `;
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+   }
+}
+
+export default new NodesSocketsRepository();

backend/src/tasks/lightning/network-sync.service.ts

@@ -10,6 +10,8 @@ import lightningApi from '../../api/lightning/lightning-api-factory';
 import nodesApi from '../../api/explorer/nodes.api';
 import { ResultSetHeader } from 'mysql2';
 import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
+import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
+import { Common } from '../../api/common';
 
 class NetworkSyncService {
   loggerTimer = 0;
@@ -58,6 +60,7 @@ class NetworkSyncService {
   private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
     let progress = 0;
 
+    let deletedSockets = 0;
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       await nodesApi.$saveNode(node);
@@ -69,8 +72,15 @@ class NetworkSyncService {
         logger.info(`Updating node ${progress}/${nodes.length}`);
         this.loggerTimer = new Date().getTime() / 1000;
       }
+
+      const addresses: string[] = [];
+      for (const socket of node.addresses) {
+        await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket));
+        addresses.push(socket.addr);
+      }
+      deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
     }
-    logger.info(`${progress} nodes updated`);
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
 
     // If a channel if not present in the graph, mark it as inactive
     nodesApi.$setNodesInactive(graphNodesPubkeys);

frontend/src/app/components/address/address-preview.component.html

@@ -44,7 +44,7 @@
     <div class="w-100 d-block d-md-none"></div>
     <div class="col-md qrcode-col">
       <div class="qr-wrapper">
-        <app-qrcode [data]="address.address" [size]="370"></app-qrcode>
+        <app-qrcode [data]="address.address" [size]="448"></app-qrcode>
       </div>
     </div>
   </div>

frontend/src/app/components/address/address-preview.component.scss

@@ -1,5 +1,5 @@
 h1 {
-  font-size: 42px;
+  font-size: 52px;
   margin: 0;
 }
 
@@ -11,23 +11,26 @@ h1 {
 }
 
 .qrcode-col {
-  width: 420px;
+  width: 468px;
-  min-width: 420px;
+  min-width: 468px;
   flex-grow: 0;
   flex-shrink: 0;
   text-align: center;
+  padding: 0;
+  margin-left: 2px;
+  margin-right: 15px;
 }
 
 .table {
-  font-size: 24px;
+  font-size: 32px;
 
   ::ng-deep .symbol {
-    font-size: 18px;
+    font-size: 24px;
   }
 }
 
 .address-link {
-  font-size: 20px;
+  font-size: 24px;
   margin-bottom: 0.5em;
   display: flex;
   flex-direction: row;
@@ -35,7 +38,7 @@ h1 {
   .truncated-address {
     text-overflow: ellipsis;
     overflow: hidden;
-    max-width: calc(505px - 4em);
+    max-width: calc(640px - 4em);
     display: inline-block;
     white-space: nowrap;
   }

frontend/src/app/components/address/address-preview.component.ts

@@ -44,7 +44,6 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
   ) { }
 
   ngOnInit() {
-    this.openGraphService.setPreviewLoading();
     this.stateService.networkChanged$.subscribe((network) => this.network = network);
 
     this.addressLoadingStatus$ = this.route.paramMap
@@ -56,6 +55,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
     this.mainSubscription = this.route.paramMap
       .pipe(
         switchMap((params: ParamMap) => {
+          this.openGraphService.waitFor('address-data');
           this.error = undefined;
           this.isLoadingAddress = true;
           this.loadedConfirmedTxCount = 0;
@@ -73,6 +73,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
                 this.isLoadingAddress = false;
                 this.error = err;
                 console.log(err);
+                this.openGraphService.fail('address-data');
                 return of(null);
               })
             );
@@ -90,7 +91,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
           this.address = address;
           this.updateChainStats();
           this.isLoadingAddress = false;
-          this.openGraphService.setPreviewReady();
+          this.openGraphService.waitOver('address-data');
         })
       )
       .subscribe(() => {},
@@ -98,6 +99,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
           console.log(error);
           this.error = error;
           this.isLoadingAddress = false;
+          this.openGraphService.fail('address-data');
         }
       );
   }

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -18,6 +18,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
   @Input() orientation = 'left';
   @Input() flip = true;
   @Output() txClickEvent = new EventEmitter<TransactionStripped>();
+  @Output() readyEvent = new EventEmitter();
 
   @ViewChild('blockCanvas')
   canvas: ElementRef<HTMLCanvasElement>;
@@ -37,6 +38,8 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
   selectedTx: TxView | void;
   tooltipPosition: Position;
 
+  readyNextFrame = false;
+
   constructor(
     readonly ngZone: NgZone,
     readonly elRef: ElementRef,
@@ -78,6 +81,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
   setup(transactions: TransactionStripped[]): void {
     if (this.scene) {
       this.scene.setup(transactions);
+      this.readyNextFrame = true;
       this.start();
     }
   }
@@ -258,6 +262,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
           this.gl.drawArrays(this.gl.TRIANGLES, 0, pointArray.length / TxSprite.vertexSize);
         }
       }
+
+      if (this.readyNextFrame) {
+        this.readyNextFrame = false;
+        this.readyEvent.emit();
+      }
     }
 
     /* LOOP */

frontend/src/app/components/block/block-preview.component.html

@@ -30,44 +30,42 @@
             <td i18n="block.weight">Weight</td>
             <td [innerHTML]="'&lrm;' + (block?.weight | wuBytes: 2)"></td>
           </tr>
-          <ng-template [ngIf]="webGlEnabled">
+          <tr *ngIf="block?.extras?.medianFee != undefined">
-            <tr *ngIf="block?.extras?.medianFee != undefined">
+            <td class="td-width" i18n="block.median-fee">Median fee</td>
-              <td class="td-width" i18n="block.median-fee">Median fee</td>
+            <td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></td>
-              <td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></td>
+          </tr>
-            </tr>
+          <ng-template [ngIf]="fees !== undefined">
-            <ng-template [ngIf]="fees !== undefined">
+            <tr>
-              <tr>
+              <td i18n="block.total-fees|Total fees in a block">Total fees</td>
-                <td i18n="block.total-fees|Total fees in a block">Total fees</td>
+              <td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
-                <td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
+                <app-amount [satoshis]="block?.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
-                  <app-amount [satoshis]="block?.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
+              </td>
+              <ng-template #liquidTotalFees>
+                <td>
+                  <app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>
                 </td>
-                <ng-template #liquidTotalFees>
+              </ng-template>
-                  <td>
-                    <app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>
-                  </td>
-                </ng-template>
-              </tr>
-            </ng-template>
-            <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
-              <td i18n="block.miner">Miner</td>
-              <td *ngIf="stateService.env.MINING_DASHBOARD">
-                <a [attr.data-cy]="'block-details-miner-badge'" placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block?.extras.pool.slug]" class="badge"
-                  [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
-                  {{ block?.extras.pool.name }}
-                </a>
-              </td>
-              <td *ngIf="!stateService.env.MINING_DASHBOARD && stateService.env.BASE_MODULE === 'mempool'">
-                <span [attr.data-cy]="'block-details-miner-badge'" placement="bottom" class="badge"
-                  [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
-                  {{ block?.extras.pool.name }}
-              </span>
-              </td>
             </tr>
           </ng-template>
+          <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
+            <td i18n="block.miner">Miner</td>
+            <td *ngIf="stateService.env.MINING_DASHBOARD">
+              <a [attr.data-cy]="'block-details-miner-badge'" placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block?.extras.pool.slug]" class="badge"
+                [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
+                {{ block?.extras.pool.name }}
+              </a>
+            </td>
+            <td *ngIf="!stateService.env.MINING_DASHBOARD && stateService.env.BASE_MODULE === 'mempool'">
+              <span [attr.data-cy]="'block-details-miner-badge'" placement="bottom" class="badge"
+                [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
+                {{ block?.extras.pool.name }}
+            </span>
+            </td>
+          </tr>
         </tbody>
       </table>
     </div>
-    <div class="col-sm chart-container" *ngIf="webGlEnabled">
+    <div class="col-sm chart-container">
       <app-block-overview-graph
         #blockGraph
         [isLoading]="false"
@@ -75,7 +73,7 @@
         [blockLimit]="stateService.blockVSize"
         [orientation]="'top'"
         [flip]="false"
-        (txClickEvent)="onTxClick($event)"
+        (readyEvent)="onGraphReady()"
       ></app-block-overview-graph>
     </div>
   </div>

frontend/src/app/components/block/block-preview.component.scss

@@ -1,23 +1,25 @@
 .block-title {
-  margin-bottom: 0.75em;
+  margin-bottom: 48px;
-  font-size: 42px;
+  font-size: 52px;
 
   ::ng-deep .next-previous-blocks {
-    font-size: 42px;
+    font-size: 52px;
   }
 }
 
 .table {
-  font-size: 24px;
+  font-size: 32px;
 }
 
 .chart-container {
   flex-grow: 0;
   flex-shrink: 0;
-  width: 420px;
+  width: 470px;
-  min-width: 420px;
+  min-width: 470px;
+  padding: 0;
+  margin-right: 15px;
 }
 
 ::ng-deep .symbol {
-  font-size: 18px;
+  font-size: 24px;
 }

frontend/src/app/components/block/block-preview.component.ts

@@ -1,11 +1,168 @@
-import { Component } from '@angular/core';
+import { Component, OnInit, OnDestroy, ViewChild, ElementRef } from '@angular/core';
-import { BlockComponent } from './block.component';
+import { ActivatedRoute, ParamMap } from '@angular/router';
+import { ElectrsApiService } from '../../services/electrs-api.service';
+import { switchMap, tap, throttleTime, catchError, shareReplay, startWith, pairwise, filter } from 'rxjs/operators';
+import { of, Subscription, asyncScheduler } from 'rxjs';
+import { StateService } from '../../services/state.service';
+import { SeoService } from 'src/app/services/seo.service';
+import { OpenGraphService } from 'src/app/services/opengraph.service';
+import { BlockExtended, TransactionStripped } from 'src/app/interfaces/node-api.interface';
+import { ApiService } from 'src/app/services/api.service';
+import { BlockOverviewGraphComponent } from 'src/app/components/block-overview-graph/block-overview-graph.component';
 
 @Component({
   selector: 'app-block-preview',
   templateUrl: './block-preview.component.html',
-  styleUrls: ['./block.component.scss', './block-preview.component.scss']
+  styleUrls: ['./block-preview.component.scss']
 })
-export class BlockPreviewComponent extends BlockComponent {
+export class BlockPreviewComponent implements OnInit, OnDestroy {
-  
+  network = '';
+  block: BlockExtended;
+  blockHeight: number;
+  blockHash: string;
+  isLoadingBlock = true;
+  strippedTransactions: TransactionStripped[];
+  overviewTransitionDirection: string;
+  isLoadingOverview = true;
+  error: any;
+  blockSubsidy: number;
+  fees: number;
+  overviewError: any = null;
+
+  overviewSubscription: Subscription;
+  networkChangedSubscription: Subscription;
+
+  @ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent;
+
+  constructor(
+    private route: ActivatedRoute,
+    private electrsApiService: ElectrsApiService,
+    public stateService: StateService,
+    private seoService: SeoService,
+    private openGraphService: OpenGraphService,
+    private apiService: ApiService
+  ) { }
+
+  ngOnInit() {
+    this.network = this.stateService.network;
+
+    const block$ = this.route.paramMap.pipe(
+      switchMap((params: ParamMap) => {
+        this.openGraphService.waitFor('block-viz');
+        this.openGraphService.waitFor('block-data');
+
+        const blockHash: string = params.get('id') || '';
+        this.block = undefined;
+        this.error = undefined;
+        this.overviewError = undefined;
+        this.fees = undefined;
+
+        let isBlockHeight = false;
+        if (/^[0-9]+$/.test(blockHash)) {
+          isBlockHeight = true;
+        } else {
+          this.blockHash = blockHash;
+        }
+
+        this.isLoadingBlock = true;
+        this.isLoadingOverview = true;
+
+        if (isBlockHeight) {
+          return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10))
+            .pipe(
+              switchMap((hash) => {
+                if (hash) {
+                  this.blockHash = hash;
+                  return this.apiService.getBlock$(hash);
+                } else {
+                  return null;
+                }
+              }),
+              catchError((err) => {
+                this.error = err;
+                this.openGraphService.fail('block-data');
+                this.openGraphService.fail('block-viz');
+                return of(null);
+              }),
+            );
+        }
+        return this.apiService.getBlock$(blockHash);
+      }),
+      filter((block: BlockExtended | void) => block != null),
+      tap((block: BlockExtended) => {
+        this.block = block;
+        this.blockHeight = block.height;
+
+        this.seoService.setTitle($localize`:@@block.component.browser-title:Block ${block.height}:BLOCK_HEIGHT:: ${block.id}:BLOCK_ID:`);
+        this.isLoadingBlock = false;
+        this.setBlockSubsidy();
+        if (block?.extras?.reward !== undefined) {
+          this.fees = block.extras.reward / 100000000 - this.blockSubsidy;
+        }
+        this.stateService.markBlock$.next({ blockHeight: this.blockHeight });
+        this.isLoadingOverview = true;
+        this.overviewError = null;
+
+        this.openGraphService.waitOver('block-data');
+      }),
+      throttleTime(50, asyncScheduler, { leading: true, trailing: true }),
+      shareReplay(1)
+    );
+
+    this.overviewSubscription = block$.pipe(
+      startWith(null),
+      pairwise(),
+      switchMap(([prevBlock, block]) => this.apiService.getStrippedBlockTransactions$(block.id)
+        .pipe(
+          catchError((err) => {
+            this.overviewError = err;
+            this.openGraphService.fail('block-viz');
+            return of([]);
+          }),
+          switchMap((transactions) => {
+            return of({ transactions, direction: 'down' });
+          })
+        )
+      ),
+    )
+    .subscribe(({transactions, direction}: {transactions: TransactionStripped[], direction: string}) => {
+      this.strippedTransactions = transactions;
+      this.isLoadingOverview = false;
+      if (this.blockGraph) {
+        this.blockGraph.destroy();
+        this.blockGraph.setup(this.strippedTransactions);
+      }
+    },
+    (error) => {
+      this.error = error;
+      this.isLoadingOverview = false;
+      this.openGraphService.fail('block-viz');
+      this.openGraphService.fail('block-data');
+      if (this.blockGraph) {
+        this.blockGraph.destroy();
+      }
+    });
+
+    this.networkChangedSubscription = this.stateService.networkChanged$
+      .subscribe((network) => this.network = network);
+  }
+
+  ngOnDestroy() {
+    if (this.overviewSubscription) {
+      this.overviewSubscription.unsubscribe();
+    }
+    if (this.networkChangedSubscription) {
+      this.networkChangedSubscription.unsubscribe();
+    }
+  }
+
+  // TODO - Refactor this.fees/this.reward for liquid because it is not
+  // used anymore on Bitcoin networks (we use block.extras directly)
+  setBlockSubsidy() {
+    this.blockSubsidy = 0;
+  }
+
+  onGraphReady(): void {
+    this.openGraphService.waitOver('block-viz');
+  }
 }

frontend/src/app/components/master-page-preview/master-page-preview.component.html

@@ -1,21 +1,20 @@
 <ng-container *ngIf="{ val: network$ | async } as network">
 <div class="preview-wrapper">
-  <router-outlet></router-outlet>
+  <header>
-
+    <span class="header-brand" style="position: relative;">
-  <footer>
+      <img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="50" width="200" class="logo">
-    <span class="footer-brand" style="position: relative;">
+      <app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 200px; height: 50px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
-      <img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="35" width="140" class="logo">
-      <app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 140px; height: 35px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
     </span>
 
     <div [ngSwitch]="network.val">
-      <span *ngSwitchCase="'signet'" class="network signet"><img src="/resources/signet-logo.png" style="width: 30px;" class="signet mr-1" alt="logo"> Signet</span>
+      <span *ngSwitchCase="'signet'" class="network signet"><img src="/resources/signet-logo.png" style="width: 45px;" class="signet mr-1" alt="logo"> Signet</span>
-      <span *ngSwitchCase="'testnet'" class="network testnet"><img src="/resources/testnet-logo.png" style="width: 30px;" class="mr-1" alt="testnet logo"> Testnet</span>
+      <span *ngSwitchCase="'testnet'" class="network testnet"><img src="/resources/testnet-logo.png" style="width: 45px;" class="mr-1" alt="testnet logo"> Testnet</span>
-      <span *ngSwitchCase="'bisq'" class="network bisq"><img src="/resources/bisq-logo.png" style="width: 30px;" class="mr-1" alt="bisq logo"> Bisq</span>
+      <span *ngSwitchCase="'bisq'" class="network bisq"><img src="/resources/bisq-logo.png" style="width: 45px;" class="mr-1" alt="bisq logo"> Bisq</span>
-      <span *ngSwitchCase="'liquid'" class="network liquid"><img src="/resources/liquid-logo.png" style="width: 30px;" class="mr-1" alt="liquid mainnet logo"> Liquid</span>
+      <span *ngSwitchCase="'liquid'" class="network liquid"><img src="/resources/liquid-logo.png" style="width: 45px;" class="mr-1" alt="liquid mainnet logo"> Liquid</span>
-      <span *ngSwitchCase="'liquidtestnet'" class="network liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 30px;" class="mr-1" alt="liquid testnet logo"> Liquid Testnet</span>
+      <span *ngSwitchCase="'liquidtestnet'" class="network liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 45px;" class="mr-1" alt="liquid testnet logo"> Liquid Testnet</span>
-      <span *ngSwitchDefault class="network mainnet"><img src="/resources/bitcoin-logo.png" style="width: 30px;" class="mainnet mr-1" alt="bitcoin logo"> Mainnet</span>
+      <span *ngSwitchDefault class="network mainnet"><img src="/resources/bitcoin-logo.png" style="width: 45px;" class="mainnet mr-1" alt="bitcoin logo"> Mainnet</span>
     </div>
-  </footer>
+  </header>
+  <router-outlet></router-outlet>
 </div>
 </ng-container>

frontend/src/app/components/master-page-preview/master-page-preview.component.scss

@@ -2,28 +2,28 @@
   position: relative;
   display: block;
   margin: auto;
-  max-width: 1024px;
+  max-width: 1200px;
-  max-height: 512px;
+  max-height: 600px;
-  padding-bottom: 64px;
+  padding-top: 80px;
 
-  footer {
+  header {
     position: absolute;
     left: 0;
     right: 0;
-    bottom: 0;
+    top: 0;
     z-index: 100;
-    min-height: 64px;
+    min-height: 80px;
-    padding: 0rem 2rem;
+    padding: 0rem 3rem;
     display: flex;
     flex-direction: row;
     justify-content: space-between;
     align-items: center;
     background: #11131f;
     text-align: start;
-    font-size: 1.2em;
+    font-size: 1.8em;
   }
 
-  .footer-brand {
+  .header-brand {
     width: 60%;
   }
 

frontend/src/app/services/opengraph.service.ts

@@ -1,4 +1,4 @@
-import { Injectable } from '@angular/core';
+import { Injectable, NgZone } from '@angular/core';
 import { Meta } from '@angular/platform-browser';
 import { Router, ActivatedRoute, NavigationEnd } from '@angular/router';
 import { filter, map, switchMap } from 'rxjs/operators';
@@ -12,8 +12,11 @@ import { LanguageService } from './language.service';
 export class OpenGraphService {
   network = '';
   defaultImageUrl = '';
+  previewLoadingEvents = {};
+  previewLoadingCount = 0;
 
   constructor(
+    private ngZone: NgZone,
     private metaService: Meta,
     private stateService: StateService,
     private LanguageService: LanguageService,
@@ -39,6 +42,11 @@ export class OpenGraphService {
         this.clearOgImage();
       }
     });
+
+    // expose routing method to global scope, so we can access it from the unfurler
+    window['ogService'] = {
+      loadPage: (path) => { return this.loadPage(path) }
+    };
   }
 
   setOgImage() {
@@ -47,8 +55,8 @@ export class OpenGraphService {
     this.metaService.updateTag({ property: 'og:image', content: ogImageUrl });
     this.metaService.updateTag({ property: 'twitter:image:src', content: ogImageUrl });
     this.metaService.updateTag({ property: 'og:image:type', content: 'image/png' });
-    this.metaService.updateTag({ property: 'og:image:width', content: '1024' });
+    this.metaService.updateTag({ property: 'og:image:width', content: '1200' });
-    this.metaService.updateTag({ property: 'og:image:height', content: '512' });
+    this.metaService.updateTag({ property: 'og:image:height', content: '600' });
   }
 
   clearOgImage() {
@@ -59,13 +67,53 @@ export class OpenGraphService {
     this.metaService.updateTag({ property: 'og:image:height', content: '500' });
   }
 
-  /// signal that the unfurler should wait for a 'ready' signal before taking a screenshot
+  /// register an event that needs to resolve before we can take a screenshot
-  setPreviewLoading() {
+  waitFor(event) {
-    this.metaService.updateTag({ property: 'og:loading', content: 'loading'});
+    if (!this.previewLoadingEvents[event]) {
+      this.previewLoadingEvents[event] = 1;
+      this.previewLoadingCount++;
+    } else {
+      this.previewLoadingEvents[event]++;
+    }
+    this.metaService.updateTag({ property: 'og:preview:loading', content: 'loading'});
   }
 
-  // signal to the unfurler that the page is ready for a screenshot
+  // mark an event as resolved
-  setPreviewReady() {
+  // if all registered events have resolved, signal we are ready for a screenshot
-    this.metaService.updateTag({ property: 'og:ready', content: 'ready'});
+  waitOver(event) {
+    if (this.previewLoadingEvents[event]) {
+      this.previewLoadingEvents[event]--;
+      if (this.previewLoadingEvents[event] === 0) {
+        delete this.previewLoadingEvents[event]
+        this.previewLoadingCount--;
+      }
+    }
+    if (this.previewLoadingCount === 0) {
+      this.metaService.updateTag({ property: 'og:preview:ready', content: 'ready'});
+    }
+  }
+
+  fail(event) {
+    if (this.previewLoadingEvents[event]) {
+      this.metaService.updateTag({ property: 'og:preview:fail', content: 'fail'});
+    }
+  }
+
+  resetLoading() {
+    this.previewLoadingEvents = {};
+    this.previewLoadingCount = 0;
+    this.metaService.removeTag("property='og:preview:loading'");
+    this.metaService.removeTag("property='og:preview:ready'");
+    this.metaService.removeTag("property='og:preview:fail'");
+    this.metaService.removeTag("property='og:meta:ready'");
+  }
+
+  loadPage(path) {
+    if (path !== this.router.url) {
+      this.resetLoading();
+      this.ngZone.run(() => {
+        this.router.navigateByUrl(path);
+      })
+    }
   }
 }

frontend/src/app/services/seo.service.ts

@@ -21,12 +21,14 @@ export class SeoService {
     this.titleService.setTitle(newTitle + ' - ' + this.getTitle());
     this.metaService.updateTag({ property: 'og:title', content: newTitle});
     this.metaService.updateTag({ property: 'twitter:title', content: newTitle});
+    this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'});
   }
 
   resetTitle(): void {
     this.titleService.setTitle(this.getTitle());
     this.metaService.updateTag({ property: 'og:title', content: this.getTitle()});
     this.metaService.updateTag({ property: 'twitter:title', content: this.getTitle()});
+    this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'});
   }
 
   setEnterpriseTitle(title: string) {

frontend/src/styles.scss

@@ -88,8 +88,8 @@ body {
 }
 
 .preview-box {
-  min-height: 512px;
+  min-height: 520px;
-  padding: 2rem 3rem;
+  padding: 1.5rem 3rem;
 }
 
 @media (max-width: 767.98px) {

unfurler/config.sample.json

@@ -5,10 +5,13 @@
   },
   "MEMPOOL": {
     "HTTP_HOST": "http://localhost",
-    "HTTP_PORT": 4200
+    "HTTP_PORT": 4200,
+    "NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin")
   },
   "PUPPETEER": {
     "CLUSTER_SIZE": 2,
-    "EXEC_PATH": "/usr/local/bin/chrome" // optional
+    "EXEC_PATH": "/usr/local/bin/chrome", // optional
+    "MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds)
+    "RENDER_TIMEOUT": 3000, // timeout for preview image rendering (in ms) (optional)
   }
 }

unfurler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-unfurl",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "Renderer for mempool open graph link preview images",
   "repository": {
     "type": "git",

unfurler/puppeteer.config.json

@@ -1,11 +1,11 @@
 {
   "headless": true,
   "defaultViewport": {
-    "width": 1024,
+    "width": 1200,
-    "height": 512
+    "height": 600
   },
   "args": [
-    "--window-size=1024,512",
+    "--window-size=1200,600",
     "--autoplay-policy=user-gesture-required",
     "--disable-background-networking",
     "--disable-background-timer-throttling",

unfurler/src/concurrency/ReusablePage.ts

@@ -0,0 +1,159 @@
+import * as puppeteer from 'puppeteer';
+import ConcurrencyImplementation from 'puppeteer-cluster/dist/concurrency/ConcurrencyImplementation';
+import { timeoutExecute } from 'puppeteer-cluster/dist/util';
+
+import config from '../config';
+const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+
+const BROWSER_TIMEOUT = 8000;
+// maximum lifetime of a single page session
+const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000;
+const maxConcurrency = config.PUPPETEER.CLUSTER_SIZE;
+
+interface RepairablePage extends puppeteer.Page {
+  repairRequested?: boolean;
+  language?: string | null;
+  createdAt?: number;
+  free?: boolean;
+  index?: number;
+}
+
+interface ResourceData {
+  page: RepairablePage;
+}
+
+export default class ReusablePage extends ConcurrencyImplementation {
+
+  protected browser: puppeteer.Browser | null = null;
+  protected pages: RepairablePage[] = [];
+  private repairing: boolean = false;
+  private repairRequested: boolean = false;
+  private openInstances: number = 0;
+  private waitingForRepairResolvers: (() => void)[] = [];
+
+  public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
+    super(options, puppeteer);
+  }
+
+  private async repair() {
+    if (this.openInstances !== 0 || this.repairing) {
+      // already repairing or there are still pages open? wait for start/finish
+      await new Promise<void>(resolve => this.waitingForRepairResolvers.push(resolve));
+      return;
+    }
+
+    this.repairing = true;
+    console.log('Starting repair');
+
+    try {
+      // will probably fail, but just in case the repair was not necessary
+      await (<puppeteer.Browser>this.browser).close();
+    } catch (e) {
+      console.log('Unable to close browser.');
+    }
+
+    try {
+      await this.init();
+    } catch (err) {
+      throw new Error('Unable to restart chrome.');
+    }
+    this.repairRequested = false;
+    this.repairing = false;
+    this.waitingForRepairResolvers.forEach(resolve => resolve());
+    this.waitingForRepairResolvers = [];
+  }
+
+  public async init() {
+    this.browser = await this.puppeteer.launch(this.options);
+    const promises = []
+    for (let i = 0; i < maxConcurrency; i++) {
+      const newPage = await this.initPage();
+      newPage.index = this.pages.length;
+      console.log('initialized page ', newPage.index);
+      this.pages.push(newPage);
+    }
+  }
+
+  public async close() {
+    await (this.browser as puppeteer.Browser).close();
+  }
+
+  protected async initPage(): Promise<RepairablePage> {
+    const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
+    page.language = null;
+    page.createdAt = Date.now();
+    const defaultUrl = mempoolHost + '/preview/block/1';
+    page.on('pageerror', (err) => {
+      page.repairRequested = true;
+    });
+    await page.goto(defaultUrl, { waitUntil: "load" });
+    page.free = true;
+    return page
+  }
+
+  protected async createResources(): Promise<ResourceData> {
+    const page = this.pages.find(p => p.free);
+    if (!page) {
+      console.log('no free pages!')
+      throw new Error('no pages available');
+    } else {
+      page.free = false;
+      return { page };
+    }
+  }
+
+  protected async repairPage(page) {
+    // create a new page
+    const newPage = await this.initPage();
+    newPage.free = true;
+    // replace the old page
+    newPage.index = page.index;
+    this.pages.splice(page.index, 1, newPage);
+    // clean up the old page
+    try {
+      await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
+    } catch (e) {
+      console.log('unexpected page repair error');
+    }
+    await page.close();
+    return newPage;
+  }
+
+  public async workerInstance() {
+    let resources: ResourceData;
+
+    return {
+      jobInstance: async () => {
+        await timeoutExecute(BROWSER_TIMEOUT, (async () => {
+          resources = await this.createResources();
+        })());
+        this.openInstances += 1;
+
+        return {
+          resources,
+
+          close: async () => {
+            this.openInstances -= 1; // decrement first in case of error
+            if (resources?.page != null) {
+              if (resources.page.repairRequested || (Date.now() - (resources.page.createdAt || 0) > maxAgeMs)) {
+                resources.page = await this.repairPage(resources.page);
+              } else {
+                resources.page.free = true;
+              }
+            }
+
+            if (this.repairRequested) {
+              await this.repair();
+            }
+          },
+        };
+      },
+
+      close: async () => {},
+
+      repair: async () => {
+        await this.repairPage(resources.page);
+      },
+    };
+  }
+}

unfurler/src/config.ts

@@ -8,10 +8,13 @@ interface IConfig {
   MEMPOOL: {
     HTTP_HOST: string;
     HTTP_PORT: number;
+    NETWORK?: string;
   };
   PUPPETEER: {
     CLUSTER_SIZE: number;
     EXEC_PATH?: string;
+    MAX_PAGE_AGE?: number;
+    RENDER_TIMEOUT?: number;
   };
 }
 

unfurler/src/index.ts

@@ -3,6 +3,8 @@ import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import config from './config';
 import { Cluster } from 'puppeteer-cluster';
+import ReusablePage from './concurrency/ReusablePage';
+import { parseLanguageUrl } from './language/lang';
 const puppeteerConfig = require('../puppeteer.config.json');
 
 if (config.PUPPETEER.EXEC_PATH) {
@@ -14,10 +16,14 @@ class Server {
   private app: Application;
   cluster?: Cluster;
   mempoolHost: string;
+  network: string;
+  defaultImageUrl: string;
 
   constructor() {
     this.app = express();
     this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+    this.network = config.MEMPOOL.NETWORK || 'bitcoin';
+    this.defaultImageUrl = this.getDefaultImageUrl();
     this.startServer();
   }
 
@@ -32,7 +38,7 @@ class Server {
       ;
 
     this.cluster = await Cluster.launch({
-        concurrency: Cluster.CONCURRENCY_CONTEXT,
+        concurrency: ReusablePage,
         maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
         puppeteerOptions: puppeteerConfig,
     });
@@ -47,63 +53,75 @@ class Server {
     });
   }
 
+  async stopServer() {
+    if (this.cluster) {
+      await this.cluster.idle();
+      await this.cluster.close();
+    }
+    if (this.server) {
+      await this.server.close();
+    }
+  }
+
   setUpRoutes() {
     this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
     this.app.get('*', (req, res) => { return this.renderHTML(req, res) })
   }
 
-  async clusterTask({ page, data: { url, action } }) {
+  async clusterTask({ page, data: { url, path, action } }) {
-    await page.goto(url, { waitUntil: "networkidle0" });
+    try {
-    switch (action) {
+      const urlParts = parseLanguageUrl(path);
-      case 'screenshot': {
+      if (page.language !== urlParts.lang) {
-        await page.evaluate(async () => {
+        // switch language
-          // wait for all images to finish loading
+        page.language = urlParts.lang;
-          const imgs = Array.from(document.querySelectorAll("img"));
+        const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
-          await Promise.all([
+        await page.goto(localizedUrl, { waitUntil: "load" });
-            document.fonts.ready,
+      } else {
-            ...imgs.map((img) => {
+        const loaded = await page.evaluate(async (path) => {
-              if (img.complete) {
+          if (window['ogService']) {
-                if (img.naturalHeight !== 0) return;
+            window['ogService'].loadPage(path);
-                throw new Error("Image failed to load");
+            return true;
-              }
+          } else {
-              return new Promise((resolve, reject) => {
+            return false;
-                img.addEventListener("load", resolve);
-                img.addEventListener("error", reject);
-              });
-            }),
-          ]);
-        });
-        const waitForReady = await page.$('meta[property="og:loading"]');
-        const alreadyReady = await page.$('meta[property="og:ready"]');
-        if (waitForReady != null && alreadyReady == null) {
-          try {
-            await page.waitForSelector('meta[property="og:ready]"', { timeout: 10000 });
-          } catch (e) {
-            // probably timed out
           }
+        }, urlParts.path);
+        if (!loaded) {
+          throw new Error('failed to access open graph service');
         }
-        return page.screenshot();
-      } break;
-      default: {
-        try {
-          await page.waitForSelector('meta[property="og:title"]', { timeout: 10000 })
-          const tag = await page.$('meta[property="og:title"]');
-        } catch (e) {
-          // probably timed out
-        }
-        return page.content();
       }
+
+      const waitForReady = await page.$('meta[property="og:preview:loading"]');
+      let success = true;
+      if (waitForReady != null) {
+        success = await Promise.race([
+          page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true),
+          page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
+        ])
+      }
+      if (success) {
+        const screenshot = await page.screenshot();
+        return screenshot;
+      } else {
+        console.log(`failed to render page preview for ${action} due to client-side error. probably requested an invalid ID`);
+        page.repairRequested = true;
+      }
+    } catch (e) {
+      console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e);
+      page.repairRequested = true;
     }
   }
 
   async renderPreview(req, res) {
     try {
-      // strip default language code for compatibility
+      const path = req.params[0]
-      const path = req.params[0].replace('/en/', '/');
+      const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' });
-      const img = await this.cluster?.execute({ url: this.mempoolHost + path, action: 'screenshot' });
 
-      res.contentType('image/png');
+      if (!img) {
-      res.send(img);
+        res.status(500).send('failed to render page preview');
+      } else {
+        res.contentType('image/png');
+        res.send(img);
+      }
     } catch (e) {
       console.log(e);
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -112,22 +130,89 @@ class Server {
 
   async renderHTML(req, res) {
     // drop requests for static files
-    const path = req.params[0];
+    const rawPath = req.params[0];
-    const match = path.match(/\.[\w]+$/);
+    const match = rawPath.match(/\.[\w]+$/);
     if (match?.length && match[0] !== '.html') {
       res.status(404).send();
-      return
+      return;
     }
 
-    try {
+    let previewSupported = true;
-      let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], action: 'html' });
+    let mode = 'mainnet'
+    let ogImageUrl = this.defaultImageUrl;
+    let ogTitle;
+    const { lang, path } = parseLanguageUrl(rawPath);
+    const parts = path.slice(1).split('/');
 
-      res.send(html)
+    // handle network mode modifiers
-    } catch (e) {
+    if (['testnet', 'signet'].includes(parts[0])) {
-      console.log(e);
+      mode = parts.shift();
-      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+
+    // handle supported preview routes
+    if (parts[0] === 'block') {
+      ogTitle = `Block: ${parts[1]}`;
+    } else if (parts[0] === 'address') {
+      ogTitle = `Address: ${parts[1]}`;
+    } else {
+      previewSupported = false;
+    }
+
+    if (previewSupported) {
+      ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`;
+      ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`;
+    } else {
+      ogTitle = 'The Mempool Open Source Project™';
+    }
+
+    res.send(`
+      <!doctype html>
+      <html lang="en-US" dir="ltr">
+      <head>
+        <meta charset="utf-8">
+        <title>${ogTitle}</title>
+        <meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the ${capitalize(this.network)} community."/>
+        <meta property="og:image" content="${ogImageUrl}"/>
+        <meta property="og:image:type" content="image/png"/>
+        <meta property="og:image:width" content="${previewSupported ? 1200 : 1000}"/>
+        <meta property="og:image:height" content="${previewSupported ? 600 : 500}"/>
+        <meta property="og:title" content="${ogTitle}">
+        <meta property="twitter:card" content="summary_large_image">
+        <meta property="twitter:site" content="@mempool">
+        <meta property="twitter:creator" content="@mempool">
+        <meta property="twitter:title" content="${ogTitle}">
+        <meta property="twitter:description" content="Our self-hosted mempool explorer for the ${capitalize(this.network)} community."/>
+        <meta property="twitter:image:src" content="${ogImageUrl}"/>
+        <meta property="twitter:domain" content="mempool.space">
+      <body></body>
+      </html>
+    `);
+  }
+
+  getDefaultImageUrl() {
+    switch (this.network) {
+      case 'liquid':
+        return this.mempoolHost + '/resources/liquid/liquid-network-preview.png';
+      case 'bisq':
+        return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png';
+      default:
+        return this.mempoolHost + '/resources/mempool-space-preview.png';
     }
   }
 }
 
 const server = new Server();
+
+process.on('SIGTERM', async () => {
+  console.info('Shutting down Mempool Unfurl Server');
+  await server.stopServer();
+  process.exit(0);
+});
+
+function capitalize(str) {
+  if (str && str.length) {
+    return str[0].toUpperCase() + str.slice(1);
+  } else {
+    return str;
+  }
+}

unfurler/src/language/lang.ts

@@ -0,0 +1,79 @@
+export interface Language {
+  code: string;
+  name: string;
+}
+
+const languageList: Language[] = [
+  { code: 'ar', name: 'العربية' },         // Arabic
+  { code: 'bg', name: 'Български' },       // Bulgarian
+  { code: 'bs', name: 'Bosanski' },        // Bosnian
+  { code: 'ca', name: 'Català' },          // Catalan
+  { code: 'cs', name: 'Čeština' },         // Czech
+  { code: 'da', name: 'Dansk' },           // Danish
+  { code: 'de', name: 'Deutsch' },         // German
+  { code: 'et', name: 'Eesti' },           // Estonian
+  { code: 'el', name: 'Ελληνικά' },        // Greek
+  { code: 'en', name: 'English' },         // English
+  { code: 'es', name: 'Español' },         // Spanish
+  { code: 'eo', name: 'Esperanto' },       // Esperanto
+  { code: 'eu', name: 'Euskara' },         // Basque
+  { code: 'fa', name: 'فارسی' },           // Persian
+  { code: 'fr', name: 'Français' },        // French
+  { code: 'gl', name: 'Galego' },          // Galician
+  { code: 'ko', name: '한국어' },          // Korean
+  { code: 'hr', name: 'Hrvatski' },        // Croatian
+  { code: 'id', name: 'Bahasa Indonesia' },// Indonesian
+  { code: 'hi', name: 'हिन्दी' },             // Hindi
+  { code: 'it', name: 'Italiano' },        // Italian
+  { code: 'he', name: 'עברית' },           // Hebrew
+  { code: 'ka', name: 'ქართული' },         // Georgian
+  { code: 'lv', name: 'Latviešu' },        // Latvian
+  { code: 'lt', name: 'Lietuvių' },        // Lithuanian
+  { code: 'hu', name: 'Magyar' },          // Hungarian
+  { code: 'mk', name: 'Македонски' },      // Macedonian
+  { code: 'ms', name: 'Bahasa Melayu' },   // Malay
+  { code: 'nl', name: 'Nederlands' },      // Dutch
+  { code: 'ja', name: '日本語' },          // Japanese
+  { code: 'nb', name: 'Norsk' },           // Norwegian Bokmål
+  { code: 'nn', name: 'Norsk Nynorsk' },   // Norwegian Nynorsk
+  { code: 'pl', name: 'Polski' },          // Polish
+  { code: 'pt', name: 'Português' },       // Portuguese
+  { code: 'pt-BR', name: 'Português (Brazil)' }, // Portuguese (Brazil)
+  { code: 'ro', name: 'Română' },          // Romanian
+  { code: 'ru', name: 'Русский' },         // Russian
+  { code: 'sk', name: 'Slovenčina' },      // Slovak
+  { code: 'sl', name: 'Slovenščina' },     // Slovenian
+  { code: 'sr', name: 'Српски / srpski' }, // Serbian
+  { code: 'sh', name: 'Srpskohrvatski / српскохрватски' },// Serbo-Croatian
+  { code: 'fi', name: 'Suomi' },           // Finnish
+  { code: 'sv', name: 'Svenska' },         // Swedish
+  { code: 'th', name: 'ไทย' },             // Thai
+  { code: 'tr', name: 'Türkçe' },          // Turkish
+  { code: 'uk', name: 'Українська' },      // Ukrainian
+  { code: 'vi', name: 'Tiếng Việt' },      // Vietnamese
+  { code: 'zh', name: '中文' },            // Chinese
+];
+
+const languageDict = {};
+languageList.forEach(lang => {
+  languageDict[lang.code] = lang
+});
+export const languages = languageDict;
+
+// expects path to start with a leading '/'
+export function parseLanguageUrl(path) {
+  const parts = path.split('/');
+  let lang;
+  let rest;
+  if (languages[parts[1]]) {
+    lang = parts[1];
+    rest = '/' + parts.slice(2).join('/');
+  } else {
+    lang = null;
+    rest = path;
+  }
+  if (lang === 'en') {
+    lang = null;
+  }
+  return { lang, path: rest };
+}