From 578a1b6d191aac6b0d4d43ee5e6650bc562ab9e9 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 2 Aug 2022 00:37:54 +0000 Subject: [PATCH 01/10] Speed up unfurls by reusing puppeteer sessions --- .../address/address-preview.component.ts | 4 +- .../block-overview-graph.component.ts | 9 ++ .../block/block-preview.component.html | 64 ++++---- .../block/block-preview.component.ts | 152 +++++++++++++++++- .../src/app/services/opengraph.service.ts | 51 +++++- frontend/src/app/services/seo.service.ts | 2 + unfurler/config.sample.json | 3 +- unfurler/src/concurrency/ReusablePage.ts | 119 ++++++++++++++ unfurler/src/config.ts | 1 + unfurler/src/index.ts | 84 +++++----- 10 files changed, 400 insertions(+), 89 deletions(-) create mode 100644 unfurler/src/concurrency/ReusablePage.ts diff --git a/frontend/src/app/components/address/address-preview.component.ts b/frontend/src/app/components/address/address-preview.component.ts index c661c29db..b762d7c9e 100644 --- a/frontend/src/app/components/address/address-preview.component.ts +++ b/frontend/src/app/components/address/address-preview.component.ts @@ -44,7 +44,6 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { ) { } ngOnInit() { - this.openGraphService.setPreviewLoading(); this.stateService.networkChanged$.subscribe((network) => this.network = network); this.addressLoadingStatus$ = this.route.paramMap @@ -56,6 +55,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.mainSubscription = this.route.paramMap .pipe( switchMap((params: ParamMap) => { + this.openGraphService.waitFor('address-data'); this.error = undefined; this.isLoadingAddress = true; this.loadedConfirmedTxCount = 0; @@ -90,7 +90,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.address = address; this.updateChainStats(); this.isLoadingAddress = false; - this.openGraphService.setPreviewReady(); + this.openGraphService.waitOver('address-data'); }) ) .subscribe(() => {}, diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 6e62a2fd0..7309a0a85 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -18,6 +18,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { @Input() orientation = 'left'; @Input() flip = true; @Output() txClickEvent = new EventEmitter(); + @Output() readyEvent = new EventEmitter(); @ViewChild('blockCanvas') canvas: ElementRef; @@ -37,6 +38,8 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { selectedTx: TxView | void; tooltipPosition: Position; + readyNextFrame = false; + constructor( readonly ngZone: NgZone, readonly elRef: ElementRef, @@ -78,6 +81,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { setup(transactions: TransactionStripped[]): void { if (this.scene) { this.scene.setup(transactions); + this.readyNextFrame = true; this.start(); } } @@ -258,6 +262,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { this.gl.drawArrays(this.gl.TRIANGLES, 0, pointArray.length / TxSprite.vertexSize); } } + + if (this.readyNextFrame) { + this.readyNextFrame = false; + this.readyEvent.emit(); + } } /* LOOP */ diff --git a/frontend/src/app/components/block/block-preview.component.html b/frontend/src/app/components/block/block-preview.component.html index c47ea236e..768bc3da3 100644 --- a/frontend/src/app/components/block/block-preview.component.html +++ b/frontend/src/app/components/block/block-preview.component.html @@ -30,44 +30,42 @@ Weight - - - Median fee - ~{{ block?.extras?.medianFee | number:'1.0-0' }} sat/vB - - - - Total fees - - + + Median fee + ~{{ block?.extras?.medianFee | number:'1.0-0' }} sat/vB + + + + Total fees + + + + + + - - - - - - - - - Miner - - - {{ block?.extras.pool.name }} - - - - - {{ block?.extras.pool.name }} - - + + + Miner + + + {{ block?.extras.pool.name }} + + + + + {{ block?.extras.pool.name }} + + + -
+
diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index bab4e0489..e59bc9c6c 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -1,11 +1,153 @@ -import { Component } from '@angular/core'; -import { BlockComponent } from './block.component'; +import { Component, OnInit, OnDestroy, ViewChild, ElementRef } from '@angular/core'; +import { ActivatedRoute, ParamMap } from '@angular/router'; +import { ElectrsApiService } from '../../services/electrs-api.service'; +import { switchMap, tap, throttleTime, catchError, shareReplay, startWith, pairwise } from 'rxjs/operators'; +import { of, Subscription, asyncScheduler } from 'rxjs'; +import { StateService } from '../../services/state.service'; +import { SeoService } from 'src/app/services/seo.service'; +import { OpenGraphService } from 'src/app/services/opengraph.service'; +import { BlockExtended, TransactionStripped } from 'src/app/interfaces/node-api.interface'; +import { ApiService } from 'src/app/services/api.service'; +import { BlockOverviewGraphComponent } from 'src/app/components/block-overview-graph/block-overview-graph.component'; @Component({ selector: 'app-block-preview', templateUrl: './block-preview.component.html', - styleUrls: ['./block.component.scss', './block-preview.component.scss'] + styleUrls: ['./block-preview.component.scss'] }) -export class BlockPreviewComponent extends BlockComponent { - +export class BlockPreviewComponent implements OnInit, OnDestroy { + network = ''; + block: BlockExtended; + blockHeight: number; + blockHash: string; + isLoadingBlock = true; + strippedTransactions: TransactionStripped[]; + overviewTransitionDirection: string; + isLoadingOverview = true; + error: any; + blockSubsidy: number; + fees: number; + overviewError: any = null; + + overviewSubscription: Subscription; + networkChangedSubscription: Subscription; + + @ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent; + + constructor( + private route: ActivatedRoute, + private electrsApiService: ElectrsApiService, + public stateService: StateService, + private seoService: SeoService, + private openGraphService: OpenGraphService, + private apiService: ApiService + ) { } + + ngOnInit() { + this.network = this.stateService.network; + + const block$ = this.route.paramMap.pipe( + switchMap((params: ParamMap) => { + this.openGraphService.waitFor('block-viz'); + this.openGraphService.waitFor('block-data'); + + const blockHash: string = params.get('id') || ''; + this.block = undefined; + this.error = undefined; + this.fees = undefined; + + let isBlockHeight = false; + if (/^[0-9]+$/.test(blockHash)) { + isBlockHeight = true; + } else { + this.blockHash = blockHash; + } + + this.isLoadingBlock = true; + this.isLoadingOverview = true; + + if (isBlockHeight) { + return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10)) + .pipe( + switchMap((hash) => { + this.blockHash = hash; + return this.apiService.getBlock$(hash); + }) + ); + } + return this.apiService.getBlock$(blockHash); + }), + tap((block: BlockExtended) => { + this.block = block; + this.blockHeight = block.height; + + this.seoService.setTitle($localize`:@@block.component.browser-title:Block ${block.height}:BLOCK_HEIGHT:: ${block.id}:BLOCK_ID:`); + this.isLoadingBlock = false; + this.setBlockSubsidy(); + if (block?.extras?.reward !== undefined) { + this.fees = block.extras.reward / 100000000 - this.blockSubsidy; + } + this.stateService.markBlock$.next({ blockHeight: this.blockHeight }); + this.isLoadingOverview = true; + this.overviewError = null; + + this.openGraphService.waitOver('block-data'); + }), + throttleTime(50, asyncScheduler, { leading: true, trailing: true }), + shareReplay(1) + ); + + this.overviewSubscription = block$.pipe( + startWith(null), + pairwise(), + switchMap(([prevBlock, block]) => this.apiService.getStrippedBlockTransactions$(block.id) + .pipe( + catchError((err) => { + this.overviewError = err; + return of([]); + }), + switchMap((transactions) => { + return of({ transactions, direction: 'down' }); + }) + ) + ), + ) + .subscribe(({transactions, direction}: {transactions: TransactionStripped[], direction: string}) => { + this.strippedTransactions = transactions; + this.isLoadingOverview = false; + if (this.blockGraph) { + this.blockGraph.destroy(); + this.blockGraph.setup(this.strippedTransactions); + } + }, + (error) => { + this.error = error; + this.isLoadingOverview = false; + if (this.blockGraph) { + this.blockGraph.destroy(); + } + }); + + this.networkChangedSubscription = this.stateService.networkChanged$ + .subscribe((network) => this.network = network); + } + + ngOnDestroy() { + if (this.overviewSubscription) { + this.overviewSubscription.unsubscribe(); + } + if (this.networkChangedSubscription) { + this.networkChangedSubscription.unsubscribe(); + } + } + + // TODO - Refactor this.fees/this.reward for liquid because it is not + // used anymore on Bitcoin networks (we use block.extras directly) + setBlockSubsidy() { + this.blockSubsidy = 0; + } + + onGraphReady(): void { + this.openGraphService.waitOver('block-viz'); + } } diff --git a/frontend/src/app/services/opengraph.service.ts b/frontend/src/app/services/opengraph.service.ts index ad62a889c..12c74efcb 100644 --- a/frontend/src/app/services/opengraph.service.ts +++ b/frontend/src/app/services/opengraph.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@angular/core'; +import { Injectable, NgZone } from '@angular/core'; import { Meta } from '@angular/platform-browser'; import { Router, ActivatedRoute, NavigationEnd } from '@angular/router'; import { filter, map, switchMap } from 'rxjs/operators'; @@ -12,8 +12,11 @@ import { LanguageService } from './language.service'; export class OpenGraphService { network = ''; defaultImageUrl = ''; + previewLoadingEvents = {}; + previewLoadingCount = 0; constructor( + private ngZone: NgZone, private metaService: Meta, private stateService: StateService, private LanguageService: LanguageService, @@ -39,6 +42,9 @@ export class OpenGraphService { this.clearOgImage(); } }); + + // expose this service to global scope, so we can access it from the unfurler + window['ogService'] = this; } setOgImage() { @@ -59,13 +65,44 @@ export class OpenGraphService { this.metaService.updateTag({ property: 'og:image:height', content: '500' }); } - /// signal that the unfurler should wait for a 'ready' signal before taking a screenshot - setPreviewLoading() { - this.metaService.updateTag({ property: 'og:loading', content: 'loading'}); + /// register an event that needs to resolve before we can take a screenshot + waitFor(event) { + if (!this.previewLoadingEvents[event]) { + this.previewLoadingEvents[event] = 1; + this.previewLoadingCount++; + } else { + this.previewLoadingEvents[event]++; + } + this.metaService.updateTag({ property: 'og:preview:loading', content: 'loading'}); } - // signal to the unfurler that the page is ready for a screenshot - setPreviewReady() { - this.metaService.updateTag({ property: 'og:ready', content: 'ready'}); + // signal that an event has resolved + // if all registered events have resolved, signal we are ready for a screenshot + waitOver(event) { + if (this.previewLoadingEvents[event]) { + this.previewLoadingEvents[event]--; + if (this.previewLoadingEvents[event] === 0) { + delete this.previewLoadingEvents[event] + this.previewLoadingCount--; + } + } + if (this.previewLoadingCount === 0) { + this.metaService.updateTag({ property: 'og:preview:ready', content: 'ready'}); + } + } + + resetLoading() { + this.previewLoadingEvents = {}; + this.previewLoadingCount = 0; + this.metaService.removeTag("property='og:preview:loading'"); + this.metaService.removeTag("property='og:preview:ready'"); + this.metaService.removeTag("property='og:meta:ready'"); + } + + loadPage(path) { + this.resetLoading(); + this.ngZone.run(() => { + this.router.navigateByUrl(path); + }) } } diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 01ed7ae8c..5f5d15c89 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -21,12 +21,14 @@ export class SeoService { this.titleService.setTitle(newTitle + ' - ' + this.getTitle()); this.metaService.updateTag({ property: 'og:title', content: newTitle}); this.metaService.updateTag({ property: 'twitter:title', content: newTitle}); + this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'}); } resetTitle(): void { this.titleService.setTitle(this.getTitle()); this.metaService.updateTag({ property: 'og:title', content: this.getTitle()}); this.metaService.updateTag({ property: 'twitter:title', content: this.getTitle()}); + this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'}); } setEnterpriseTitle(title: string) { diff --git a/unfurler/config.sample.json b/unfurler/config.sample.json index 02f2b78f0..c48f6f5b2 100644 --- a/unfurler/config.sample.json +++ b/unfurler/config.sample.json @@ -9,6 +9,7 @@ }, "PUPPETEER": { "CLUSTER_SIZE": 2, - "EXEC_PATH": "/usr/local/bin/chrome" // optional + "EXEC_PATH": "/usr/local/bin/chrome", // optional + "MAX_PAGE_AGE": 86400 // maximum lifetime of a page session (in seconds) } } diff --git a/unfurler/src/concurrency/ReusablePage.ts b/unfurler/src/concurrency/ReusablePage.ts new file mode 100644 index 000000000..98cdadc4d --- /dev/null +++ b/unfurler/src/concurrency/ReusablePage.ts @@ -0,0 +1,119 @@ +import * as puppeteer from 'puppeteer'; +import ConcurrencyImplementation, { ResourceData } from 'puppeteer-cluster/dist/concurrency/ConcurrencyImplementation'; +import { timeoutExecute } from 'puppeteer-cluster/dist/util'; + +import config from '../config'; +const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + +const BROWSER_TIMEOUT = 5000; +// maximum lifetime of a single page session +const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000; + +interface repairablePage extends puppeteer.Page { + repairRequested?: boolean; +} + +export default class ReusablePage extends ConcurrencyImplementation { + + protected browser: puppeteer.Browser | null = null; + protected currentPage: repairablePage | null = null; + protected pageCreatedAt: number = 0; + private repairing: boolean = false; + private repairRequested: boolean = false; + private openInstances: number = 0; + private waitingForRepairResolvers: (() => void)[] = []; + + public constructor(options: puppeteer.LaunchOptions, puppeteer: any) { + super(options, puppeteer); + } + + private async repair() { + if (this.openInstances !== 0 || this.repairing) { + // already repairing or there are still pages open? wait for start/finish + await new Promise(resolve => this.waitingForRepairResolvers.push(resolve)); + return; + } + + this.repairing = true; + console.log('Starting repair'); + + try { + // will probably fail, but just in case the repair was not necessary + await (this.browser).close(); + } catch (e) { + console.log('Unable to close browser.'); + } + + try { + this.browser = await this.puppeteer.launch(this.options) as puppeteer.Browser; + } catch (err) { + throw new Error('Unable to restart chrome.'); + } + this.currentPage = null; + this.repairRequested = false; + this.repairing = false; + this.waitingForRepairResolvers.forEach(resolve => resolve()); + this.waitingForRepairResolvers = []; + await this.createResources(); + } + + public async init() { + this.browser = await this.puppeteer.launch(this.options); + } + + public async close() { + await (this.browser as puppeteer.Browser).close(); + } + + protected async createResources(): Promise { + if (!this.currentPage) { + this.currentPage = await (this.browser as puppeteer.Browser).newPage(); + this.pageCreatedAt = Date.now(); + const defaultUrl = mempoolHost + '/preview/block/1'; + this.currentPage.on('pageerror', (err) => { + this.repairRequested = true; + }); + await this.currentPage.goto(defaultUrl, { waitUntil: "load" }); + } + return { + page: this.currentPage + } + } + + public async workerInstance() { + let resources: ResourceData; + + return { + jobInstance: async () => { + if (this.repairRequested || this.currentPage?.repairRequested) { + await this.repair(); + } + + await timeoutExecute(BROWSER_TIMEOUT, (async () => { + resources = await this.createResources(); + })()); + this.openInstances += 1; + + return { + resources, + + close: async () => { + this.openInstances -= 1; // decrement first in case of error + + if (this.repairRequested || this.currentPage?.repairRequested || (Date.now() - this.pageCreatedAt > maxAgeMs)) { + await this.repair(); + } + }, + }; + }, + + close: async () => {}, + + repair: async () => { + console.log('Repair requested'); + this.repairRequested = true; + await this.repair(); + }, + }; + } +} diff --git a/unfurler/src/config.ts b/unfurler/src/config.ts index 1df60ce98..dd77eae56 100644 --- a/unfurler/src/config.ts +++ b/unfurler/src/config.ts @@ -12,6 +12,7 @@ interface IConfig { PUPPETEER: { CLUSTER_SIZE: number; EXEC_PATH?: string; + MAX_PAGE_AGE?: number; }; } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 49815fcb1..d84ce883a 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -3,6 +3,7 @@ import { Application, Request, Response, NextFunction } from 'express'; import * as http from 'http'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; +import ReusablePage from './concurrency/ReusablePage'; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -32,7 +33,7 @@ class Server { ; this.cluster = await Cluster.launch({ - concurrency: Cluster.CONCURRENCY_CONTEXT, + concurrency: ReusablePage, maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, puppeteerOptions: puppeteerConfig, }); @@ -52,47 +53,40 @@ class Server { this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } - async clusterTask({ page, data: { url, action } }) { - await page.goto(url, { waitUntil: "networkidle0" }); - switch (action) { - case 'screenshot': { - await page.evaluate(async () => { - // wait for all images to finish loading - const imgs = Array.from(document.querySelectorAll("img")); - await Promise.all([ - document.fonts.ready, - ...imgs.map((img) => { - if (img.complete) { - if (img.naturalHeight !== 0) return; - throw new Error("Image failed to load"); - } - return new Promise((resolve, reject) => { - img.addEventListener("load", resolve); - img.addEventListener("error", reject); - }); - }), - ]); - }); - const waitForReady = await page.$('meta[property="og:loading"]'); - const alreadyReady = await page.$('meta[property="og:ready"]'); - if (waitForReady != null && alreadyReady == null) { - try { - await page.waitForSelector('meta[property="og:ready]"', { timeout: 10000 }); - } catch (e) { - // probably timed out + async clusterTask({ page, data: { url, path, action } }) { + try { + if (action === 'screenshot' || action === 'html') { + const loaded = await page.evaluate(async (path) => { + if (window['ogService']) { + window['ogService'].loadPage(path); + return true; + } else { + return false; } + }, path) + + if (!loaded) { + throw new Error('failed to access open graph service'); } - return page.screenshot(); - } break; - default: { - try { - await page.waitForSelector('meta[property="og:title"]', { timeout: 10000 }) - const tag = await page.$('meta[property="og:title"]'); - } catch (e) { - // probably timed out + + if (action === 'screenshot') { + const waitForReady = await page.$('meta[property="og:preview:loading"]'); + const alreadyReady = await page.$('meta[property="og:preview:ready"]'); + if (waitForReady != null && alreadyReady == null) { + await page.waitForSelector('meta[property="og:preview:ready"]', { timeout: 8000 }); + } + return page.screenshot(); + } else if (action === 'html') { + const alreadyReady = await page.$('meta[property="og:meta:ready"]'); + if (alreadyReady == null) { + await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: 8000 }); + } + return page.content(); } - return page.content(); } + } catch (e) { + console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e); + page.repairRequested = true; } } @@ -100,8 +94,11 @@ class Server { try { // strip default language code for compatibility const path = req.params[0].replace('/en/', '/'); - const img = await this.cluster?.execute({ url: this.mempoolHost + path, action: 'screenshot' }); + const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' }); + if (!img) { + throw new Error('failed to render preview image'); + } res.contentType('image/png'); res.send(img); } catch (e) { @@ -120,9 +117,14 @@ class Server { } try { - let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], action: 'html' }); + // strip default language code for compatibility + const path = req.params[0].replace('/en/', '/'); - res.send(html) + let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], path: req.params[0], action: 'html' }); + if (!html) { + throw new Error('failed to render preview image'); + } + res.send(html); } catch (e) { console.log(e); res.status(500).send(e instanceof Error ? e.message : e); From e4342113fa45215eca73b8ee58d952d490e4630a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 2 Aug 2022 01:15:50 +0000 Subject: [PATCH 02/10] Improve unfurl layout & resize to 1200x600 --- .../address/address-preview.component.html | 2 +- .../address/address-preview.component.scss | 17 +++++++------ .../block/block-preview.component.scss | 16 ++++++------ .../master-page-preview.component.html | 25 +++++++++---------- .../master-page-preview.component.scss | 18 ++++++------- frontend/src/styles.scss | 4 +-- unfurler/puppeteer.config.json | 6 ++--- 7 files changed, 46 insertions(+), 42 deletions(-) diff --git a/frontend/src/app/components/address/address-preview.component.html b/frontend/src/app/components/address/address-preview.component.html index bc73d064b..30b9c29e6 100644 --- a/frontend/src/app/components/address/address-preview.component.html +++ b/frontend/src/app/components/address/address-preview.component.html @@ -44,7 +44,7 @@
- +
diff --git a/frontend/src/app/components/address/address-preview.component.scss b/frontend/src/app/components/address/address-preview.component.scss index f286c6ca1..2de368547 100644 --- a/frontend/src/app/components/address/address-preview.component.scss +++ b/frontend/src/app/components/address/address-preview.component.scss @@ -1,5 +1,5 @@ h1 { - font-size: 42px; + font-size: 52px; margin: 0; } @@ -11,23 +11,26 @@ h1 { } .qrcode-col { - width: 420px; - min-width: 420px; + width: 468px; + min-width: 468px; flex-grow: 0; flex-shrink: 0; text-align: center; + padding: 0; + margin-left: 2px; + margin-right: 15px; } .table { - font-size: 24px; + font-size: 32px; ::ng-deep .symbol { - font-size: 18px; + font-size: 24px; } } .address-link { - font-size: 20px; + font-size: 24px; margin-bottom: 0.5em; display: flex; flex-direction: row; @@ -35,7 +38,7 @@ h1 { .truncated-address { text-overflow: ellipsis; overflow: hidden; - max-width: calc(505px - 4em); + max-width: calc(640px - 4em); display: inline-block; white-space: nowrap; } diff --git a/frontend/src/app/components/block/block-preview.component.scss b/frontend/src/app/components/block/block-preview.component.scss index f2049a1d3..2c1f40bc5 100644 --- a/frontend/src/app/components/block/block-preview.component.scss +++ b/frontend/src/app/components/block/block-preview.component.scss @@ -1,23 +1,25 @@ .block-title { - margin-bottom: 0.75em; - font-size: 42px; + margin-bottom: 48px; + font-size: 52px; ::ng-deep .next-previous-blocks { - font-size: 42px; + font-size: 52px; } } .table { - font-size: 24px; + font-size: 32px; } .chart-container { flex-grow: 0; flex-shrink: 0; - width: 420px; - min-width: 420px; + width: 470px; + min-width: 470px; + padding: 0; + margin-right: 15px; } ::ng-deep .symbol { - font-size: 18px; + font-size: 24px; } diff --git a/frontend/src/app/components/master-page-preview/master-page-preview.component.html b/frontend/src/app/components/master-page-preview/master-page-preview.component.html index 6c2e45242..52a3e7026 100644 --- a/frontend/src/app/components/master-page-preview/master-page-preview.component.html +++ b/frontend/src/app/components/master-page-preview/master-page-preview.component.html @@ -1,21 +1,20 @@
- - -
- - - +
+ + +
- logo Signet - testnet logo Testnet - bisq logo Bisq - liquid mainnet logo Liquid - liquid testnet logo Liquid Testnet - bitcoin logo Mainnet + logo Signet + testnet logo Testnet + bisq logo Bisq + liquid mainnet logo Liquid + liquid testnet logo Liquid Testnet + bitcoin logo Mainnet
-
+ +
diff --git a/frontend/src/app/components/master-page-preview/master-page-preview.component.scss b/frontend/src/app/components/master-page-preview/master-page-preview.component.scss index 0384e0f86..605c4f6d9 100644 --- a/frontend/src/app/components/master-page-preview/master-page-preview.component.scss +++ b/frontend/src/app/components/master-page-preview/master-page-preview.component.scss @@ -2,28 +2,28 @@ position: relative; display: block; margin: auto; - max-width: 1024px; - max-height: 512px; - padding-bottom: 64px; + max-width: 1200px; + max-height: 600px; + padding-top: 80px; - footer { + header { position: absolute; left: 0; right: 0; - bottom: 0; + top: 0; z-index: 100; - min-height: 64px; - padding: 0rem 2rem; + min-height: 80px; + padding: 0rem 3rem; display: flex; flex-direction: row; justify-content: space-between; align-items: center; background: #11131f; text-align: start; - font-size: 1.2em; + font-size: 1.8em; } - .footer-brand { + .header-brand { width: 60%; } diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index da4bdcffe..2ef537456 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -88,8 +88,8 @@ body { } .preview-box { - min-height: 512px; - padding: 2rem 3rem; + min-height: 520px; + padding: 1.5rem 3rem; } @media (max-width: 767.98px) { diff --git a/unfurler/puppeteer.config.json b/unfurler/puppeteer.config.json index 346deb1b7..3de7b0652 100644 --- a/unfurler/puppeteer.config.json +++ b/unfurler/puppeteer.config.json @@ -1,11 +1,11 @@ { "headless": true, "defaultViewport": { - "width": 1024, - "height": 512 + "width": 1200, + "height": 600 }, "args": [ - "--window-size=1024,512", + "--window-size=1200,600", "--autoplay-policy=user-gesture-required", "--disable-background-networking", "--disable-background-timer-throttling", From 06f232fdd8a99011ab26b2d5eff0105990225943 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 2 Aug 2022 01:46:23 +0000 Subject: [PATCH 03/10] handle SIGTERM gracefully in unfurler --- unfurler/src/index.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index d84ce883a..089c9a280 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -48,6 +48,16 @@ class Server { }); } + async stopServer() { + if (this.cluster) { + await this.cluster.idle(); + await this.cluster.close(); + } + if (this.server) { + await this.server.close(); + } + } + setUpRoutes() { this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) @@ -133,3 +143,9 @@ class Server { } const server = new Server(); + +process.on('SIGTERM', async () => { + console.info('Shutting down Mempool Unfurl Server'); + await server.stopServer(); + process.exit(0); +}); From 94d1aeb287d91f695232f1ac6543ca4a58ed92b6 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 2 Aug 2022 21:02:33 +0000 Subject: [PATCH 04/10] Fix unfurler language support --- .../src/app/services/opengraph.service.ts | 12 +-- unfurler/src/concurrency/ReusablePage.ts | 8 +- unfurler/src/index.ts | 54 +++++++------ unfurler/src/language/lang.ts | 79 +++++++++++++++++++ 4 files changed, 121 insertions(+), 32 deletions(-) create mode 100644 unfurler/src/language/lang.ts diff --git a/frontend/src/app/services/opengraph.service.ts b/frontend/src/app/services/opengraph.service.ts index 12c74efcb..58de73325 100644 --- a/frontend/src/app/services/opengraph.service.ts +++ b/frontend/src/app/services/opengraph.service.ts @@ -76,7 +76,7 @@ export class OpenGraphService { this.metaService.updateTag({ property: 'og:preview:loading', content: 'loading'}); } - // signal that an event has resolved + // mark an event as resolved // if all registered events have resolved, signal we are ready for a screenshot waitOver(event) { if (this.previewLoadingEvents[event]) { @@ -100,9 +100,11 @@ export class OpenGraphService { } loadPage(path) { - this.resetLoading(); - this.ngZone.run(() => { - this.router.navigateByUrl(path); - }) + if (path !== this.router.url) { + this.resetLoading(); + this.ngZone.run(() => { + this.router.navigateByUrl(path); + }) + } } } diff --git a/unfurler/src/concurrency/ReusablePage.ts b/unfurler/src/concurrency/ReusablePage.ts index 98cdadc4d..c8f40b2c8 100644 --- a/unfurler/src/concurrency/ReusablePage.ts +++ b/unfurler/src/concurrency/ReusablePage.ts @@ -5,12 +5,13 @@ import { timeoutExecute } from 'puppeteer-cluster/dist/util'; import config from '../config'; const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); -const BROWSER_TIMEOUT = 5000; +const BROWSER_TIMEOUT = 8000; // maximum lifetime of a single page session const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000; interface repairablePage extends puppeteer.Page { repairRequested?: boolean; + language?: string | null; } export default class ReusablePage extends ConcurrencyImplementation { @@ -68,6 +69,7 @@ export default class ReusablePage extends ConcurrencyImplementation { protected async createResources(): Promise { if (!this.currentPage) { this.currentPage = await (this.browser as puppeteer.Browser).newPage(); + this.currentPage.language = null; this.pageCreatedAt = Date.now(); const defaultUrl = mempoolHost + '/preview/block/1'; this.currentPage.on('pageerror', (err) => { @@ -85,10 +87,6 @@ export default class ReusablePage extends ConcurrencyImplementation { return { jobInstance: async () => { - if (this.repairRequested || this.currentPage?.repairRequested) { - await this.repair(); - } - await timeoutExecute(BROWSER_TIMEOUT, (async () => { resources = await this.createResources(); })()); diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 089c9a280..54db5fa97 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -4,6 +4,7 @@ import * as http from 'http'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; +import { parseLanguageUrl } from './language/lang'; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -46,6 +47,8 @@ class Server { this.server.listen(config.SERVER.HTTP_PORT, () => { console.log(`Mempool Unfurl Server is running on port ${config.SERVER.HTTP_PORT}`); }); + + this.initClusterPages(); } async stopServer() { @@ -63,9 +66,24 @@ class Server { this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } + async initClusterPages() { + for (let i = 0; i < config.PUPPETEER.CLUSTER_SIZE; i++) { + this.cluster?.execute({ action: 'init' }); + } + } + async clusterTask({ page, data: { url, path, action } }) { + if (action === 'init') { + return; + } try { - if (action === 'screenshot' || action === 'html') { + const urlParts = parseLanguageUrl(path); + if (page.language !== urlParts.lang) { + // switch language + page.language = urlParts.lang; + const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ; + await page.goto(localizedUrl, { waitUntil: "load" }); + } else { const loaded = await page.evaluate(async (path) => { if (window['ogService']) { window['ogService'].loadPage(path); @@ -73,26 +91,21 @@ class Server { } else { return false; } - }, path) - + }, urlParts.path); if (!loaded) { throw new Error('failed to access open graph service'); } + } - if (action === 'screenshot') { - const waitForReady = await page.$('meta[property="og:preview:loading"]'); - const alreadyReady = await page.$('meta[property="og:preview:ready"]'); - if (waitForReady != null && alreadyReady == null) { - await page.waitForSelector('meta[property="og:preview:ready"]', { timeout: 8000 }); - } - return page.screenshot(); - } else if (action === 'html') { - const alreadyReady = await page.$('meta[property="og:meta:ready"]'); - if (alreadyReady == null) { - await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: 8000 }); - } - return page.content(); + if (action === 'screenshot') { + const waitForReady = await page.$('meta[property="og:preview:loading"]'); + if (waitForReady != null) { + await page.waitForSelector('meta[property="og:preview:ready"]', { timeout: 3000 }); } + return page.screenshot(); + } else if (action === 'html') { + await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: 3000 }); + return page.content(); } } catch (e) { console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e); @@ -102,13 +115,13 @@ class Server { async renderPreview(req, res) { try { - // strip default language code for compatibility - const path = req.params[0].replace('/en/', '/'); + const path = req.params[0] const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' }); if (!img) { throw new Error('failed to render preview image'); } + res.contentType('image/png'); res.send(img); } catch (e) { @@ -127,10 +140,7 @@ class Server { } try { - // strip default language code for compatibility - const path = req.params[0].replace('/en/', '/'); - - let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], path: req.params[0], action: 'html' }); + let html = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'html' }); if (!html) { throw new Error('failed to render preview image'); } diff --git a/unfurler/src/language/lang.ts b/unfurler/src/language/lang.ts new file mode 100644 index 000000000..610e68312 --- /dev/null +++ b/unfurler/src/language/lang.ts @@ -0,0 +1,79 @@ +export interface Language { + code: string; + name: string; +} + +const languageList: Language[] = [ + { code: 'ar', name: 'العربية' }, // Arabic + { code: 'bg', name: 'Български' }, // Bulgarian + { code: 'bs', name: 'Bosanski' }, // Bosnian + { code: 'ca', name: 'Català' }, // Catalan + { code: 'cs', name: 'Čeština' }, // Czech + { code: 'da', name: 'Dansk' }, // Danish + { code: 'de', name: 'Deutsch' }, // German + { code: 'et', name: 'Eesti' }, // Estonian + { code: 'el', name: 'Ελληνικά' }, // Greek + { code: 'en', name: 'English' }, // English + { code: 'es', name: 'Español' }, // Spanish + { code: 'eo', name: 'Esperanto' }, // Esperanto + { code: 'eu', name: 'Euskara' }, // Basque + { code: 'fa', name: 'فارسی' }, // Persian + { code: 'fr', name: 'Français' }, // French + { code: 'gl', name: 'Galego' }, // Galician + { code: 'ko', name: '한국어' }, // Korean + { code: 'hr', name: 'Hrvatski' }, // Croatian + { code: 'id', name: 'Bahasa Indonesia' },// Indonesian + { code: 'hi', name: 'हिन्दी' }, // Hindi + { code: 'it', name: 'Italiano' }, // Italian + { code: 'he', name: 'עברית' }, // Hebrew + { code: 'ka', name: 'ქართული' }, // Georgian + { code: 'lv', name: 'Latviešu' }, // Latvian + { code: 'lt', name: 'Lietuvių' }, // Lithuanian + { code: 'hu', name: 'Magyar' }, // Hungarian + { code: 'mk', name: 'Македонски' }, // Macedonian + { code: 'ms', name: 'Bahasa Melayu' }, // Malay + { code: 'nl', name: 'Nederlands' }, // Dutch + { code: 'ja', name: '日本語' }, // Japanese + { code: 'nb', name: 'Norsk' }, // Norwegian Bokmål + { code: 'nn', name: 'Norsk Nynorsk' }, // Norwegian Nynorsk + { code: 'pl', name: 'Polski' }, // Polish + { code: 'pt', name: 'Português' }, // Portuguese + { code: 'pt-BR', name: 'Português (Brazil)' }, // Portuguese (Brazil) + { code: 'ro', name: 'Română' }, // Romanian + { code: 'ru', name: 'Русский' }, // Russian + { code: 'sk', name: 'Slovenčina' }, // Slovak + { code: 'sl', name: 'Slovenščina' }, // Slovenian + { code: 'sr', name: 'Српски / srpski' }, // Serbian + { code: 'sh', name: 'Srpskohrvatski / српскохрватски' },// Serbo-Croatian + { code: 'fi', name: 'Suomi' }, // Finnish + { code: 'sv', name: 'Svenska' }, // Swedish + { code: 'th', name: 'ไทย' }, // Thai + { code: 'tr', name: 'Türkçe' }, // Turkish + { code: 'uk', name: 'Українська' }, // Ukrainian + { code: 'vi', name: 'Tiếng Việt' }, // Vietnamese + { code: 'zh', name: '中文' }, // Chinese +]; + +const languageDict = {}; +languageList.forEach(lang => { + languageDict[lang.code] = lang +}); +export const languages = languageDict; + +// expects path to start with a leading '/' +export function parseLanguageUrl(path) { + const parts = path.split('/'); + let lang; + let rest; + if (languages[parts[1]]) { + lang = parts[1]; + rest = '/' + parts.slice(2).join('/'); + } else { + lang = null; + rest = path; + } + if (lang === 'en') { + lang = null; + } + return { lang, path: rest }; +} From 31ced9e23cd325a8d302270c1d7b86d0c1175f2e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 3 Aug 2022 01:11:54 +0000 Subject: [PATCH 05/10] don't use puppeteer to render unfurl meta tags --- .../src/app/services/opengraph.service.ts | 4 +- unfurler/config.sample.json | 6 +- unfurler/src/config.ts | 2 + unfurler/src/index.ts | 100 ++++++++++++++---- 4 files changed, 87 insertions(+), 25 deletions(-) diff --git a/frontend/src/app/services/opengraph.service.ts b/frontend/src/app/services/opengraph.service.ts index 58de73325..50f84fa11 100644 --- a/frontend/src/app/services/opengraph.service.ts +++ b/frontend/src/app/services/opengraph.service.ts @@ -53,8 +53,8 @@ export class OpenGraphService { this.metaService.updateTag({ property: 'og:image', content: ogImageUrl }); this.metaService.updateTag({ property: 'twitter:image:src', content: ogImageUrl }); this.metaService.updateTag({ property: 'og:image:type', content: 'image/png' }); - this.metaService.updateTag({ property: 'og:image:width', content: '1024' }); - this.metaService.updateTag({ property: 'og:image:height', content: '512' }); + this.metaService.updateTag({ property: 'og:image:width', content: '1200' }); + this.metaService.updateTag({ property: 'og:image:height', content: '600' }); } clearOgImage() { diff --git a/unfurler/config.sample.json b/unfurler/config.sample.json index c48f6f5b2..e080ee68a 100644 --- a/unfurler/config.sample.json +++ b/unfurler/config.sample.json @@ -5,11 +5,13 @@ }, "MEMPOOL": { "HTTP_HOST": "http://localhost", - "HTTP_PORT": 4200 + "HTTP_PORT": 4200, + "NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin") }, "PUPPETEER": { "CLUSTER_SIZE": 2, "EXEC_PATH": "/usr/local/bin/chrome", // optional - "MAX_PAGE_AGE": 86400 // maximum lifetime of a page session (in seconds) + "MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds) + "RENDER_TIMEOUT": 3000, // timeout for preview image rendering (in ms) (optional) } } diff --git a/unfurler/src/config.ts b/unfurler/src/config.ts index dd77eae56..a65d48f6f 100644 --- a/unfurler/src/config.ts +++ b/unfurler/src/config.ts @@ -8,11 +8,13 @@ interface IConfig { MEMPOOL: { HTTP_HOST: string; HTTP_PORT: number; + NETWORK?: string; }; PUPPETEER: { CLUSTER_SIZE: number; EXEC_PATH?: string; MAX_PAGE_AGE?: number; + RENDER_TIMEOUT?: number; }; } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 54db5fa97..f9ce6fd31 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -16,10 +16,14 @@ class Server { private app: Application; cluster?: Cluster; mempoolHost: string; + network: string; + defaultImageUrl: string; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + this.network = config.MEMPOOL.NETWORK || 'bitcoin'; + this.defaultImageUrl = this.getDefaultImageUrl(); this.startServer(); } @@ -97,16 +101,11 @@ class Server { } } - if (action === 'screenshot') { - const waitForReady = await page.$('meta[property="og:preview:loading"]'); - if (waitForReady != null) { - await page.waitForSelector('meta[property="og:preview:ready"]', { timeout: 3000 }); - } - return page.screenshot(); - } else if (action === 'html') { - await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: 3000 }); - return page.content(); + const waitForReady = await page.$('meta[property="og:preview:loading"]'); + if (waitForReady != null) { + await page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }); } + return page.screenshot(); } catch (e) { console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e); page.repairRequested = true; @@ -132,22 +131,73 @@ class Server { async renderHTML(req, res) { // drop requests for static files - const path = req.params[0]; - const match = path.match(/\.[\w]+$/); + const rawPath = req.params[0]; + const match = rawPath.match(/\.[\w]+$/); if (match?.length && match[0] !== '.html') { res.status(404).send(); - return + return; } - try { - let html = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'html' }); - if (!html) { - throw new Error('failed to render preview image'); - } - res.send(html); - } catch (e) { - console.log(e); - res.status(500).send(e instanceof Error ? e.message : e); + let previewSupported = true; + let mode = 'mainnet' + let ogImageUrl = this.defaultImageUrl; + let ogTitle; + const { lang, path } = parseLanguageUrl(rawPath); + const parts = path.slice(1).split('/'); + + // handle network mode modifiers + if (['testnet', 'signet'].includes(parts[0])) { + mode = parts.shift(); + } + + // handle supported preview routes + if (parts[0] === 'block') { + ogTitle = `Block: ${parts[1]}`; + } else if (parts[0] === 'address') { + ogTitle = `Address: ${parts[1]}`; + } else { + previewSupported = false; + } + + if (previewSupported) { + ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; + ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`; + } else { + ogTitle = 'The Mempool Open Source Project™'; + } + + res.send(` + + + + + ${ogTitle} + + + + + + + + + + + + + + + + `); + } + + getDefaultImageUrl() { + switch (this.network) { + case 'liquid': + return '/resources/liquid/liquid-network-preview.png'; + case 'bisq': + return '/resources/bisq/bisq-markets-preview.png'; + default: + return '/resources/mempool-space-preview.png'; } } } @@ -159,3 +209,11 @@ process.on('SIGTERM', async () => { await server.stopServer(); process.exit(0); }); + +function capitalize(str) { + if (str && str.length) { + return str[0].toUpperCase() + str.slice(1); + } else { + return str; + } +} From bbf04648f95e9169369992bdad9600b8e44dcb37 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 3 Aug 2022 16:43:47 +0000 Subject: [PATCH 06/10] Handle missing blocks/addresses in preview --- .../address/address-preview.component.ts | 2 ++ .../block/block-preview.component.ts | 23 +++++++++++++++---- .../src/app/services/opengraph.service.ts | 6 +++-- unfurler/package.json | 2 +- unfurler/src/index.ts | 6 ++--- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/frontend/src/app/components/address/address-preview.component.ts b/frontend/src/app/components/address/address-preview.component.ts index b762d7c9e..37ee44e2e 100644 --- a/frontend/src/app/components/address/address-preview.component.ts +++ b/frontend/src/app/components/address/address-preview.component.ts @@ -73,6 +73,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.isLoadingAddress = false; this.error = err; console.log(err); + this.openGraphService.waitOver('address-data'); return of(null); }) ); @@ -98,6 +99,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { console.log(error); this.error = error; this.isLoadingAddress = false; + this.openGraphService.waitOver('address-data'); } ); } diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index e59bc9c6c..dd8ca170f 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit, OnDestroy, ViewChild, ElementRef } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; -import { switchMap, tap, throttleTime, catchError, shareReplay, startWith, pairwise } from 'rxjs/operators'; +import { switchMap, tap, throttleTime, catchError, shareReplay, startWith, pairwise, filter } from 'rxjs/operators'; import { of, Subscription, asyncScheduler } from 'rxjs'; import { StateService } from '../../services/state.service'; import { SeoService } from 'src/app/services/seo.service'; @@ -54,6 +54,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { const blockHash: string = params.get('id') || ''; this.block = undefined; this.error = undefined; + this.overviewError = undefined; this.fees = undefined; let isBlockHeight = false; @@ -70,13 +71,24 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10)) .pipe( switchMap((hash) => { - this.blockHash = hash; - return this.apiService.getBlock$(hash); - }) + if (hash) { + this.blockHash = hash; + return this.apiService.getBlock$(hash); + } else { + return null; + } + }), + catchError((err) => { + this.error = err; + this.openGraphService.waitOver('block-data'); + this.openGraphService.waitOver('block-viz'); + return of(null); + }), ); } return this.apiService.getBlock$(blockHash); }), + filter((block: BlockExtended | void) => block != null), tap((block: BlockExtended) => { this.block = block; this.blockHeight = block.height; @@ -104,6 +116,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { .pipe( catchError((err) => { this.overviewError = err; + this.openGraphService.waitOver('block-viz'); return of([]); }), switchMap((transactions) => { @@ -123,6 +136,8 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { (error) => { this.error = error; this.isLoadingOverview = false; + this.openGraphService.waitOver('block-viz'); + this.openGraphService.waitOver('block-data'); if (this.blockGraph) { this.blockGraph.destroy(); } diff --git a/frontend/src/app/services/opengraph.service.ts b/frontend/src/app/services/opengraph.service.ts index 50f84fa11..9ed57b9e8 100644 --- a/frontend/src/app/services/opengraph.service.ts +++ b/frontend/src/app/services/opengraph.service.ts @@ -43,8 +43,10 @@ export class OpenGraphService { } }); - // expose this service to global scope, so we can access it from the unfurler - window['ogService'] = this; + // expose routing method to global scope, so we can access it from the unfurler + window['ogService'] = { + loadPage: (path) => { return this.loadPage(path) } + }; } setOgImage() { diff --git a/unfurler/package.json b/unfurler/package.json index 0d6d938d6..2d353bfdf 100644 --- a/unfurler/package.json +++ b/unfurler/package.json @@ -1,6 +1,6 @@ { "name": "mempool-unfurl", - "version": "0.0.1", + "version": "0.0.2", "description": "Renderer for mempool open graph link preview images", "repository": { "type": "git", diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index f9ce6fd31..d97bd652d 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -193,11 +193,11 @@ class Server { getDefaultImageUrl() { switch (this.network) { case 'liquid': - return '/resources/liquid/liquid-network-preview.png'; + return this.mempoolHost + '/resources/liquid/liquid-network-preview.png'; case 'bisq': - return '/resources/bisq/bisq-markets-preview.png'; + return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png'; default: - return '/resources/mempool-space-preview.png'; + return this.mempoolHost + '/resources/mempool-space-preview.png'; } } } From 0631f357b699ed3340f463727a8c143ab22caddd Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 12 Aug 2022 16:34:41 +0000 Subject: [PATCH 07/10] Improve unfurler client-side error handling --- .../address/address-preview.component.ts | 4 ++-- .../block/block-preview.component.ts | 10 ++++----- .../src/app/services/opengraph.service.ts | 7 ++++++ unfurler/src/index.ts | 22 ++++++++++++++----- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/frontend/src/app/components/address/address-preview.component.ts b/frontend/src/app/components/address/address-preview.component.ts index 37ee44e2e..c0f6fff81 100644 --- a/frontend/src/app/components/address/address-preview.component.ts +++ b/frontend/src/app/components/address/address-preview.component.ts @@ -73,7 +73,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.isLoadingAddress = false; this.error = err; console.log(err); - this.openGraphService.waitOver('address-data'); + this.openGraphService.fail('address-data'); return of(null); }) ); @@ -99,7 +99,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { console.log(error); this.error = error; this.isLoadingAddress = false; - this.openGraphService.waitOver('address-data'); + this.openGraphService.fail('address-data'); } ); } diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index dd8ca170f..f1c7216e1 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -80,8 +80,8 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { }), catchError((err) => { this.error = err; - this.openGraphService.waitOver('block-data'); - this.openGraphService.waitOver('block-viz'); + this.openGraphService.fail('block-data'); + this.openGraphService.fail('block-viz'); return of(null); }), ); @@ -116,7 +116,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { .pipe( catchError((err) => { this.overviewError = err; - this.openGraphService.waitOver('block-viz'); + this.openGraphService.fail('block-viz'); return of([]); }), switchMap((transactions) => { @@ -136,8 +136,8 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { (error) => { this.error = error; this.isLoadingOverview = false; - this.openGraphService.waitOver('block-viz'); - this.openGraphService.waitOver('block-data'); + this.openGraphService.fail('block-viz'); + this.openGraphService.fail('block-data'); if (this.blockGraph) { this.blockGraph.destroy(); } diff --git a/frontend/src/app/services/opengraph.service.ts b/frontend/src/app/services/opengraph.service.ts index 9ed57b9e8..dc62db0f3 100644 --- a/frontend/src/app/services/opengraph.service.ts +++ b/frontend/src/app/services/opengraph.service.ts @@ -93,11 +93,18 @@ export class OpenGraphService { } } + fail(event) { + if (this.previewLoadingEvents[event]) { + this.metaService.updateTag({ property: 'og:preview:fail', content: 'fail'}); + } + } + resetLoading() { this.previewLoadingEvents = {}; this.previewLoadingCount = 0; this.metaService.removeTag("property='og:preview:loading'"); this.metaService.removeTag("property='og:preview:ready'"); + this.metaService.removeTag("property='og:preview:fail'"); this.metaService.removeTag("property='og:meta:ready'"); } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index d97bd652d..81edd9325 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -102,10 +102,20 @@ class Server { } const waitForReady = await page.$('meta[property="og:preview:loading"]'); + let success = true; if (waitForReady != null) { - await page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }); + success = await Promise.race([ + page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true), + page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false) + ]) + } + if (success) { + const screenshot = await page.screenshot(); + return screenshot; + } else { + console.log(`failed to render page preview for ${action} due to client-side error. probably requested an invalid ID`); + page.repairRequested = true; } - return page.screenshot(); } catch (e) { console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e); page.repairRequested = true; @@ -118,11 +128,11 @@ class Server { const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' }); if (!img) { - throw new Error('failed to render preview image'); + res.status(500).send('failed to render page preview'); + } else { + res.contentType('image/png'); + res.send(img); } - - res.contentType('image/png'); - res.send(img); } catch (e) { console.log(e); res.status(500).send(e instanceof Error ? e.message : e); From 1a903d3efb9c108a2a2a4411633b94572ded4108 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 12 Aug 2022 16:41:55 +0000 Subject: [PATCH 08/10] Unbork unfurler concurrency implementation --- unfurler/src/concurrency/ReusablePage.ts | 86 ++++++++++++++++++------ unfurler/src/index.ts | 11 --- 2 files changed, 64 insertions(+), 33 deletions(-) diff --git a/unfurler/src/concurrency/ReusablePage.ts b/unfurler/src/concurrency/ReusablePage.ts index c8f40b2c8..9592ea702 100644 --- a/unfurler/src/concurrency/ReusablePage.ts +++ b/unfurler/src/concurrency/ReusablePage.ts @@ -1,5 +1,5 @@ import * as puppeteer from 'puppeteer'; -import ConcurrencyImplementation, { ResourceData } from 'puppeteer-cluster/dist/concurrency/ConcurrencyImplementation'; +import ConcurrencyImplementation from 'puppeteer-cluster/dist/concurrency/ConcurrencyImplementation'; import { timeoutExecute } from 'puppeteer-cluster/dist/util'; import config from '../config'; @@ -8,17 +8,24 @@ const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + const BROWSER_TIMEOUT = 8000; // maximum lifetime of a single page session const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000; +const maxConcurrency = config.PUPPETEER.CLUSTER_SIZE; -interface repairablePage extends puppeteer.Page { +interface RepairablePage extends puppeteer.Page { repairRequested?: boolean; language?: string | null; + createdAt?: number; + free?: boolean; + index?: number; +} + +interface ResourceData { + page: RepairablePage; } export default class ReusablePage extends ConcurrencyImplementation { protected browser: puppeteer.Browser | null = null; - protected currentPage: repairablePage | null = null; - protected pageCreatedAt: number = 0; + protected pages: RepairablePage[] = []; private repairing: boolean = false; private repairRequested: boolean = false; private openInstances: number = 0; @@ -46,40 +53,70 @@ export default class ReusablePage extends ConcurrencyImplementation { } try { - this.browser = await this.puppeteer.launch(this.options) as puppeteer.Browser; + await this.init(); } catch (err) { throw new Error('Unable to restart chrome.'); } - this.currentPage = null; this.repairRequested = false; this.repairing = false; this.waitingForRepairResolvers.forEach(resolve => resolve()); this.waitingForRepairResolvers = []; - await this.createResources(); } public async init() { this.browser = await this.puppeteer.launch(this.options); + const promises = [] + for (let i = 0; i < maxConcurrency; i++) { + const newPage = await this.initPage(); + newPage.index = this.pages.length; + console.log('initialized page ', newPage.index); + this.pages.push(newPage); + } } public async close() { await (this.browser as puppeteer.Browser).close(); } + protected async initPage(): Promise { + const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage; + page.language = null; + page.createdAt = Date.now(); + const defaultUrl = mempoolHost + '/preview/block/1'; + page.on('pageerror', (err) => { + page.repairRequested = true; + }); + await page.goto(defaultUrl, { waitUntil: "load" }); + page.free = true; + return page + } + protected async createResources(): Promise { - if (!this.currentPage) { - this.currentPage = await (this.browser as puppeteer.Browser).newPage(); - this.currentPage.language = null; - this.pageCreatedAt = Date.now(); - const defaultUrl = mempoolHost + '/preview/block/1'; - this.currentPage.on('pageerror', (err) => { - this.repairRequested = true; - }); - await this.currentPage.goto(defaultUrl, { waitUntil: "load" }); + const page = this.pages.find(p => p.free); + if (!page) { + console.log('no free pages!') + throw new Error('no pages available'); + } else { + page.free = false; + return { page }; } - return { - page: this.currentPage + } + + protected async repairPage(page) { + // create a new page + const newPage = await this.initPage(); + newPage.free = true; + // replace the old page + newPage.index = page.index; + this.pages.splice(page.index, 1, newPage); + // clean up the old page + try { + await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?) + } catch (e) { + console.log('unexpected page repair error'); } + await page.close(); + return newPage; } public async workerInstance() { @@ -97,8 +134,15 @@ export default class ReusablePage extends ConcurrencyImplementation { close: async () => { this.openInstances -= 1; // decrement first in case of error + if (resources?.page != null) { + if (resources.page.repairRequested || (Date.now() - (resources.page.createdAt || 0) > maxAgeMs)) { + resources.page = await this.repairPage(resources.page); + } else { + resources.page.free = true; + } + } - if (this.repairRequested || this.currentPage?.repairRequested || (Date.now() - this.pageCreatedAt > maxAgeMs)) { + if (this.repairRequested) { await this.repair(); } }, @@ -108,9 +152,7 @@ export default class ReusablePage extends ConcurrencyImplementation { close: async () => {}, repair: async () => { - console.log('Repair requested'); - this.repairRequested = true; - await this.repair(); + await this.repairPage(resources.page); }, }; } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 81edd9325..ca85ae5cc 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -51,8 +51,6 @@ class Server { this.server.listen(config.SERVER.HTTP_PORT, () => { console.log(`Mempool Unfurl Server is running on port ${config.SERVER.HTTP_PORT}`); }); - - this.initClusterPages(); } async stopServer() { @@ -70,16 +68,7 @@ class Server { this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } - async initClusterPages() { - for (let i = 0; i < config.PUPPETEER.CLUSTER_SIZE; i++) { - this.cluster?.execute({ action: 'init' }); - } - } - async clusterTask({ page, data: { url, path, action } }) { - if (action === 'init') { - return; - } try { const urlParts = parseLanguageUrl(path); if (page.language !== urlParts.lang) { From 3e9543f0b6f7b6dc0f2b4c7ae91f0c0aa6fbbf7e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 13 Aug 2022 10:24:11 +0200 Subject: [PATCH 09/10] Create and populate nodes_socket table --- backend/src/api/common.ts | 33 ++++++++++++++ backend/src/api/database-migration.ts | 18 +++++++- .../clightning/clightning-convert.ts | 2 +- .../api/lightning/lightning-api.interface.ts | 2 +- .../repositories/BlocksAuditsRepository.ts | 1 - .../repositories/NodesSocketsRepository.ts | 45 +++++++++++++++++++ .../tasks/lightning/network-sync.service.ts | 12 ++++- 7 files changed, 107 insertions(+), 6 deletions(-) create mode 100644 backend/src/repositories/NodesSocketsRepository.ts diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 8d9de53c9..fe12e0f40 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,5 +1,7 @@ import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; import config from '../config'; +import { NodeSocket } from '../repositories/NodesSocketsRepository'; +import { isIP } from 'net'; export class Common { static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ? '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49' @@ -221,4 +223,35 @@ export class Common { const d = new Date((date || 0) * 1000); return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; } + + static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket { + let network: string | null = null; + + if (config.LIGHTNING.BACKEND === 'cln') { + network = socket.network; + } else if (config.LIGHTNING.BACKEND === 'lnd') { + if (socket.addr.indexOf('onion') !== -1) { + if (socket.addr.split('.')[0].length >= 56) { + network = 'torv3'; + } else { + network = 'torv2'; + } + } else if (socket.addr.indexOf('i2p') !== -1) { + network = 'i2p'; + } else { + const ipv = isIP(socket.addr.split(':')[0]); + if (ipv === 4) { + network = 'ipv4'; + } else if (ipv === 6) { + network = 'ipv6'; + } + } + } + + return { + publicKey: publicKey, + network: network, + addr: socket.addr, + }; + } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index cfc0092d8..050829c5a 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 36; + private static currentVersion = 37; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -324,6 +324,10 @@ class DatabaseMigration { if (databaseSchemaVersion < 36 && isBitcoin == true) { await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"'); } + + if (databaseSchemaVersion < 37 && isBitcoin == true) { + await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets')); + } } /** @@ -737,7 +741,7 @@ class DatabaseMigration { names text DEFAULT NULL, UNIQUE KEY id (id,type), KEY id_2 (id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } private getCreateBlocksPricesTableQuery(): string { @@ -749,6 +753,16 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateLNNodesSocketsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS nodes_sockets ( + public_key varchar(66) NOT NULL, + socket varchar(100) NOT NULL, + type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns') NULL, + UNIQUE KEY public_key_socket (public_key, socket), + INDEX (public_key) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 15d8d8766..656c3c6da 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -17,7 +17,7 @@ export function convertNode(clNode: any): ILightningApi.Node { network: addr.type, addr: `${addr.address}:${addr.port}` }; - }), + }) ?? [], last_update: clNode?.last_timestamp ?? 0, }; } diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index 283f34a5a..1a5e2793f 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -82,4 +82,4 @@ export namespace ILightningApi { is_required: boolean; is_known: boolean; } -} +} \ No newline at end of file diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 54b723959..be85b22b9 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -1,4 +1,3 @@ -import transactionUtils from '../api/transaction-utils'; import DB from '../database'; import logger from '../logger'; import { BlockAudit } from '../mempool.interfaces'; diff --git a/backend/src/repositories/NodesSocketsRepository.ts b/backend/src/repositories/NodesSocketsRepository.ts new file mode 100644 index 000000000..af594e6e1 --- /dev/null +++ b/backend/src/repositories/NodesSocketsRepository.ts @@ -0,0 +1,45 @@ +import { ResultSetHeader } from 'mysql2'; +import DB from '../database'; +import logger from '../logger'; + +export interface NodeSocket { + publicKey: string; + network: string | null; + addr: string; +} + +class NodesSocketsRepository { + public async $saveSocket(socket: NodeSocket): Promise { + try { + await DB.query(` + INSERT INTO nodes_sockets(public_key, socket, type) + VALUE (?, ?, ?) + `, [socket.publicKey, socket.addr, socket.network]); + } catch (e: any) { + if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this + logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e)); + // We don't throw, not a critical issue if we miss some nodes sockets + } + } + } + + public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise { + if (addresses.length === 0) { + return 0; + } + try { + const query = ` + DELETE FROM nodes_sockets + WHERE public_key = ? + AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')}) + `; + const [result] = await DB.query(query, [publicKey]); + return result.affectedRows; + } catch (e) { + logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e)); + return 0; + } + } +} + +export default new NodesSocketsRepository(); diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 1fdd77361..f0122c5ca 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -10,6 +10,8 @@ import lightningApi from '../../api/lightning/lightning-api-factory'; import nodesApi from '../../api/explorer/nodes.api'; import { ResultSetHeader } from 'mysql2'; import fundingTxFetcher from './sync-tasks/funding-tx-fetcher'; +import NodesSocketsRepository from '../../repositories/NodesSocketsRepository'; +import { Common } from '../../api/common'; class NetworkSyncService { loggerTimer = 0; @@ -58,6 +60,7 @@ class NetworkSyncService { private async $updateNodesList(nodes: ILightningApi.Node[]): Promise { let progress = 0; + let deletedSockets = 0; const graphNodesPubkeys: string[] = []; for (const node of nodes) { await nodesApi.$saveNode(node); @@ -69,8 +72,15 @@ class NetworkSyncService { logger.info(`Updating node ${progress}/${nodes.length}`); this.loggerTimer = new Date().getTime() / 1000; } + + const addresses: string[] = []; + for (const socket of node.addresses) { + await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket)); + addresses.push(socket.addr); + } + deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses); } - logger.info(`${progress} nodes updated`); + logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`); // If a channel if not present in the graph, mark it as inactive nodesApi.$setNodesInactive(graphNodesPubkeys); From ebb119aa901647d8fc81b9c6fbca68689edf91cc Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 13 Aug 2022 11:01:18 +0200 Subject: [PATCH 10/10] Add 'websocket' type to nodes sockets --- backend/src/api/database-migration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 050829c5a..f3512248f 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -757,7 +757,7 @@ class DatabaseMigration { return `CREATE TABLE IF NOT EXISTS nodes_sockets ( public_key varchar(66) NOT NULL, socket varchar(100) NOT NULL, - type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns') NULL, + type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL, UNIQUE KEY public_key_socket (public_key, socket), INDEX (public_key) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;