mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-02-24 14:50:52 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/menu

Browse source
This commit is contained in:
nymkappa 2023-08-20 08:21:38 +02:00 • committed by GitHub
commit ef554ad67b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 76 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
backend/src/api/difficulty-adjustment.ts
View file

@ -32,13 +32,13 @@ export interface DifficultyAdjustment {
export function calcBitsDifference(oldBits: number, newBits: number): number {
// Must be
// - integer
// - highest exponent is 0x1f, so max value (as integer) is 0x1f0000ff
// - highest exponent is 0x20, so max value (as integer) is 0x207fffff
// - min value is 1 (exponent = 0)
// - highest bit of the number-part is +- sign, it must not be 1
const verifyBits = (bits: number): void => {
if (
Math.floor(bits) !== bits ||
bits > 0x1f0000ff ||
bits > 0x207fffff ||
bits < 1 ||
(bits & 0x00800000) !== 0 ||
(bits & 0x007fffff) === 0

23
frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts
View file

@ -31,6 +31,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
@Input() spotlight: number = 0;
@Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`;
@Input() allBlocks: boolean = false;
@Input() forceRtl: boolean = false;
mempoolWidth: number = 0;
@Output() widthChange: EventEmitter<number> = new EventEmitter();
@ -102,7 +103,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
}
this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => {
this.timeLtr = !!ltr;
this.timeLtr = !this.forceRtl && !!ltr;
this.cd.markForCheck();
});
@ -114,11 +115,6 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
});
this.reduceEmptyBlocksToFitScreen(this.mempoolEmptyBlocks);
this.mempoolBlocks.map(() => {
this.updateMempoolBlockStyles();
this.calculateTransactionPosition();
});
this.reduceMempoolBlocksToFitScreen(this.mempoolBlocks);
this.isTabHiddenSubscription = this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden);
this.loadingBlocks$ = combineLatest([
this.stateService.isLoadingWebSocket$,
@ -206,14 +202,17 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
if (!block) {
return;
}
const isNewBlock = block.height > this.chainTip;
if (this.chainTip === -1) {
this.animateEntry = block.height === this.stateService.latestBlockHeight;
} else {
this.animateEntry = block.height > this.chainTip;
this.animateEntry = isNewBlock;
}
this.chainTip = this.stateService.latestBlockHeight;
if ((block?.extras?.similarity == null || block?.extras?.similarity > 0.5) && !this.tabHidden) {
if (isNewBlock && (block?.extras?.similarity == null || block?.extras?.similarity > 0.5) && !this.tabHidden) {
this.blockIndex++;
}
});
@ -283,7 +282,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
}
reduceEmptyBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {
const innerWidth = this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2;
const innerWidth = this.containerWidth || (this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2);
let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT;
if (!this.allBlocks) {
blocksAmount = Math.min(this.stateService.env.MEMPOOL_BLOCKS_AMOUNT, Math.floor(innerWidth / (this.blockWidth + this.blockPadding)));
@ -306,7 +305,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
}
reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {
const innerWidth = this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2;
const innerWidth = this.containerWidth || (this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2);
let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT;
if (this.count) {
blocksAmount = 8;
@ -316,7 +315,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
while (blocks.length > blocksAmount) {
const block = blocks.pop();
if (!this.count) {
const lastBlock = blocks[0];
const lastBlock = blocks[blocks.length - 1];
lastBlock.blockSize += block.blockSize;
lastBlock.blockVSize += block.blockVSize;
lastBlock.nTx += block.nTx;
@ -327,7 +326,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
}
}
if (blocks.length) {
blocks[0].isStack = blocks[0].blockVSize > this.stateService.blockVSize;
blocks[blocks.length - 1].isStack = blocks[blocks.length - 1].blockVSize > this.stateService.blockVSize;
}
return blocks;
}

11
production/install
View file

@ -530,6 +530,7 @@ osCertbotDryRun()
zfsCreateFilesystems()
{
zfs create -o "mountpoint=/backup" "${ZPOOL}/backup"
zfs create -o "mountpoint=/var/cache/nginx" "${ZPOOL}/cache"
zfs create -o "mountpoint=${ELEMENTS_HOME}" "${ZPOOL}/elements"
zfs create -o "mountpoint=${BITCOIN_HOME}" "${ZPOOL}/bitcoin"
@ -1641,19 +1642,11 @@ fi
################################################
if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
echo "[*] Installing Elements Liquid Testnet electrs start script"
osSudo "${ROOT_USER}" install -c -o "${ELEMENTS_USER}" -g "${ELEMENTS_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-liquidtestnet" "${ELEMENTS_ELECTRS_HOME}"
echo "[*] Installing Elements Liquid Testnet RPC credentials"
osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${ELEMENTS_HOME}/elements.conf"
osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${ELEMENTS_HOME}/elements.conf"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_HOME}/elements.conf"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_HOME}/elements.conf"
echo "[*] Configuring Elements LiquidTestnet RPC credentials in electrs start script"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquidtestnet"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquidtestnet"
osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquidtestnet"
fi
################################
@ -1860,8 +1853,6 @@ chown "${MEMPOOL_USER}:${MEMPOOL_GROUP}" "${MEMPOOL_MYSQL_CREDENTIALS}"
echo "[*] Adding Nginx configuration"
osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
mkdir -p /var/cache/nginx/services /var/cache/nginx/api
chown "${NGINX_USER}:${NGINX_GROUP}" /var/cache/nginx/services /var/cache/nginx/api
ln -s "${MEMPOOL_HOME}/mempool" "${NGINX_ETC_FOLDER}/mempool"
osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}"
osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"

12
production/nginx/http-proxy-cache.conf
View file

@ -1,7 +1,7 @@
# proxy cache
proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=600s max_size=200m;
types_hash_max_size 2048;
proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=365d max_size=2000m;
proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=365d max_size=2000m;
proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=365d max_size=5000m;
proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=365d max_size=100m;
proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=365d max_size=100m;
types_hash_max_size 4096;

8
production/nginx/server-common.conf
View file

@ -136,8 +136,10 @@ location @mempool-space-unfurler {
proxy_cache_background_update on;
proxy_cache_use_stale updating;
proxy_cache unfurler;
proxy_cache_valid 200 10m;
proxy_cache_valid 200 1h; # will re-render page if older than this
proxy_redirect off;
expires 1d;
}
location @mempool-space-slurper {
@ -151,6 +153,8 @@ location @mempool-space-slurper {
proxy_cache_background_update on;
proxy_cache_use_stale updating;
proxy_cache slurper;
proxy_cache_valid 200 10m;
proxy_cache_valid 200 1h; # will re-render page if older than this
proxy_redirect off;
expires 10d;
}

9
unfurler/src/concurrency/ReusablePage.ts
View file

@ -11,12 +11,13 @@ const BROWSER_TIMEOUT = 8000;
const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000;
const maxConcurrency = config.PUPPETEER.CLUSTER_SIZE;
interface RepairablePage extends puppeteer.Page {
export interface RepairablePage extends puppeteer.Page {
repairRequested?: boolean;
language?: string | null;
createdAt?: number;
free?: boolean;
index?: number;
clusterGroup?: string;
}
interface ResourceData {
@ -76,7 +77,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
for (let i = 0; i < maxConcurrency; i++) {
const newPage = await this.initPage();
newPage.index = this.pages.length;
logger.info(`initialized page ${newPage.index}`);
logger.info(`initialized page ${newPage.clusterGroup}:${newPage.index}`);
this.pages.push(newPage);
}
}
@ -87,6 +88,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
protected async initPage(): Promise<RepairablePage> {
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
page.clusterGroup = 'unfurler';
page.language = null;
page.createdAt = Date.now();
let defaultUrl
@ -129,6 +131,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
protected async repairPage(page) {
// create a new page
logger.debug(`Repairing page ${page.clusterGroup}:${page.index}`);
const newPage = await this.initPage();
newPage.free = true;
// replace the old page
@ -138,7 +141,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
try {
await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
} catch (e) {
logger.err('unexpected page repair error');
logger.err(`unexpected page repair error ${page.clusterGroup}:${page.index}`);
}
await page.close();
return newPage;

13
unfurler/src/concurrency/ReusableSSRPage.ts
View file

@ -2,19 +2,11 @@ import * as puppeteer from 'puppeteer';
import { timeoutExecute } from 'puppeteer-cluster/dist/util';
import logger from '../logger';
import config from '../config';
import ReusablePage from './ReusablePage';
import ReusablePage, { RepairablePage } from './ReusablePage';
const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64');
interface RepairablePage extends puppeteer.Page {
repairRequested?: boolean;
language?: string | null;
createdAt?: number;
free?: boolean;
index?: number;
}
export default class ReusableSSRPage extends ReusablePage {
public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
@ -27,13 +19,14 @@ export default class ReusableSSRPage extends ReusablePage {
protected async initPage(): Promise<RepairablePage> {
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
page.clusterGroup = 'slurper';
page.language = null;
page.createdAt = Date.now();
const defaultUrl = mempoolHost + '/about';
page.on('pageerror', (err) => {
console.log(err);
// page.repairRequested = true;
page.repairRequested = true;
});
await page.setRequestInterception(true);
page.on('request', req => {

38
unfurler/src/index.ts
View file

@ -5,7 +5,7 @@ import * as https from 'https';
import config from './config';
import { Cluster } from 'puppeteer-cluster';
import ReusablePage from './concurrency/ReusablePage';
import ReusableSSRPage from './concurrency/ReusablePage';
import ReusableSSRPage from './concurrency/ReusableSSRPage';
import { parseLanguageUrl } from './language/lang';
import { matchRoute } from './routes';
import nodejsPath from 'path';
@ -120,8 +120,9 @@ class Server {
this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) })
}
async clusterTask({ page, data: { url, path, action } }) {
async clusterTask({ page, data: { url, path, action, reqUrl } }) {
try {
logger.info(`rendering "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
const urlParts = parseLanguageUrl(path);
if (page.language !== urlParts.lang) {
// switch language
@ -156,20 +157,21 @@ class Server {
});
return screenshot;
} else if (success === false) {
logger.warn(`failed to render ${path} for ${action} due to client-side error, e.g. requested an invalid txid`);
logger.warn(`failed to render ${reqUrl} for ${action} due to client-side error, e.g. requested an invalid txid`);
page.repairRequested = true;
} else {
logger.warn(`failed to render ${path} for ${action} due to puppeteer timeout`);
logger.warn(`failed to render ${reqUrl} for ${action} due to puppeteer timeout`);
page.repairRequested = true;
}
} catch (e) {
logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true;
}
}
async ssrClusterTask({ page, data: { url, path, action } }) {
async ssrClusterTask({ page, data: { url, path, action, reqUrl } }) {
try {
logger.info(`slurping "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
const urlParts = parseLanguageUrl(path);
if (page.language !== urlParts.lang) {
// switch language
@ -207,7 +209,7 @@ class Server {
let html = await page.content();
return html;
} else {
logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true;
}
}
@ -228,7 +230,9 @@ class Server {
// don't bother unless the route is definitely renderable
if (rawPath.includes('/preview/') && matchedRoute.render) {
img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' });
img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot', reqUrl: req.url });
} else {
logger.info('rendering not enabled for page "' + req.url + '"');
}
if (!img) {
@ -258,10 +262,17 @@ class Server {
res.status(404).send();
return;
} else {
logger.info('proxying resource "' + req.url + '"');
if (this.secureHost) {
https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
res.writeHead(got.statusCode, got.headers);
return got.pipe(res);
});
} else {
http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
res.writeHead(got.statusCode, got.headers);
return got.pipe(res);
});
}
return;
}
@ -270,9 +281,10 @@ class Server {
let result = '';
try {
if (unfurl) {
logger.info('unfurling "' + req.url + '"');
result = await this.renderUnfurlMeta(rawPath);
} else {
result = await this.renderSEOPage(rawPath);
result = await this.renderSEOPage(rawPath, req.url);
}
if (result && result.length) {
if (result === '404') {
@ -326,8 +338,8 @@ class Server {
</html>`;
}
async renderSEOPage(rawPath: string): Promise<string> {
let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' });
async renderSEOPage(rawPath: string, reqUrl: string): Promise<string> {
let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr', reqUrl });
// remove javascript to prevent double hydration
if (html && html.length) {
html = html.replaceAll(/<script.*<\/script>/g, "");