mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-02-23 22:46:54 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4152 from mempool/mononaut/unfurler-mode-headers

Browse source 
Select unfurl mode with route prefix
This commit is contained in:
wiz 2023-08-17 16:48:26 +09:00 • committed by GitHub
commit 487d5de0ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 236 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

288
production/nginx-cache-warmer
View file

@ -1,140 +1,192 @@
#!/usr/bin/env zsh
delay=0.15
hostname=$(hostname)
slugs=(`curl -sSL https://${hostname}/api/v1/mining/pools/3y|jq -r -S '(.pools[].slug)'`)
warmSlurp()
{
echo "$1"
curl -i -s -H 'User-Agent: Googlebot' "$1" | head -1
}
warmUnfurl()
{
echo "$1"
curl -i -s -H 'User-Agent: Twitterbot' "$1" | head -1
}
warm()
{
echo "$1"
curl -i -s "$1" | head -1
}
while true
do for url in / \
'/api/v1/blocks' \
'/api/v1/statistics/2h' \
'/api/v1/statistics/24h' \
'/api/v1/statistics/1w' \
'/api/v1/statistics/1m' \
'/api/v1/statistics/3m' \
'/api/v1/statistics/6m' \
'/api/v1/statistics/1y' \
'/api/v1/statistics/2y' \
'/api/v1/statistics/3y' \
'/api/v1/statistics/4y' \
'/api/v1/statistics/all' \
'/api/v1/mining/pools/24h' \
'/api/v1/mining/pools/3d' \
'/api/v1/mining/pools/1w' \
'/api/v1/mining/pools/1m' \
'/api/v1/mining/pools/3m' \
'/api/v1/mining/pools/6m' \
'/api/v1/mining/pools/1y' \
'/api/v1/mining/pools/2y' \
'/api/v1/mining/pools/3y' \
'/api/v1/mining/pools/all' \
'/api/v1/mining/hashrate/3m' \
'/api/v1/mining/hashrate/6m' \
'/api/v1/mining/hashrate/1y' \
'/api/v1/mining/hashrate/2y' \
'/api/v1/mining/hashrate/3y' \
'/api/v1/mining/hashrate/all' \
'/api/v1/mining/hashrate/pools/3m' \
'/api/v1/mining/hashrate/pools/6m' \
'/api/v1/mining/hashrate/pools/1y' \
'/api/v1/mining/hashrate/pools/2y' \
'/api/v1/mining/hashrate/pools/3y' \
'/api/v1/mining/hashrate/pools/all' \
'/api/v1/mining/reward-stats/144' \
'/api/v1/mining/blocks/fees/24h' \
'/api/v1/mining/blocks/fees/3d' \
'/api/v1/mining/blocks/fees/1w' \
'/api/v1/mining/blocks/fees/1m' \
'/api/v1/mining/blocks/fees/3m' \
'/api/v1/mining/blocks/fees/6m' \
'/api/v1/mining/blocks/fees/1y' \
'/api/v1/mining/blocks/fees/2y' \
'/api/v1/mining/blocks/fees/3y' \
'/api/v1/mining/blocks/fees/all' \
'/api/v1/mining/blocks/rewards/24h' \
'/api/v1/mining/blocks/rewards/3d' \
'/api/v1/mining/blocks/rewards/1w' \
'/api/v1/mining/blocks/rewards/1m' \
'/api/v1/mining/blocks/rewards/3m' \
'/api/v1/mining/blocks/rewards/6m' \
'/api/v1/mining/blocks/rewards/1y' \
'/api/v1/mining/blocks/rewards/2y' \
'/api/v1/mining/blocks/rewards/3y' \
'/api/v1/mining/blocks/rewards/all' \
'/api/v1/mining/blocks/fee-rates/24h' \
'/api/v1/mining/blocks/fee-rates/3d' \
'/api/v1/mining/blocks/fee-rates/1w' \
'/api/v1/mining/blocks/fee-rates/1m' \
'/api/v1/mining/blocks/fee-rates/3m' \
'/api/v1/mining/blocks/fee-rates/6m' \
'/api/v1/mining/blocks/fee-rates/1y' \
'/api/v1/mining/blocks/fee-rates/2y' \
'/api/v1/mining/blocks/fee-rates/3y' \
'/api/v1/mining/blocks/fee-rates/all' \
'/api/v1/mining/difficulty-adjustments/24h' \
'/api/v1/mining/difficulty-adjustments/3d' \
'/api/v1/mining/difficulty-adjustments/1w' \
'/api/v1/mining/difficulty-adjustments/1m' \
'/api/v1/mining/difficulty-adjustments/3m' \
'/api/v1/mining/difficulty-adjustments/6m' \
'/api/v1/mining/difficulty-adjustments/1y' \
'/api/v1/mining/difficulty-adjustments/2y' \
'/api/v1/mining/difficulty-adjustments/3y' \
'/api/v1/mining/difficulty-adjustments/all' \
'/api/v1/lightning/channels-geo?style=widget' \
'/api/v1/lightning/channels-geo?style=graph' \
'/api/v1/lightning/statistics/latest' \
'/api/v1/lightning/statistics/1m' \
'/api/v1/lightning/statistics/3m' \
'/api/v1/lightning/statistics/6m' \
'/api/v1/lightning/statistics/1y' \
'/api/v1/lightning/statistics/2y' \
'/api/v1/lightning/statistics/3y' \
'/api/v1/lightning/statistics/all' \
'/api/v1/lightning/nodes/isp-ranking' \
'/api/v1/lightning/nodes/isp/15169,396982' `# Google` \
'/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` \
'/api/v1/lightning/nodes/isp/39572' `# DataWeb` \
'/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \
'/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \
'/api/v1/lightning/nodes/isp/394745' `# LunaNode` \
'/api/v1/lightning/nodes/isp/45102' `# Alibaba` \
'/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \
'/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \
'/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` \
'/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` \
'/api/v1/lightning/nodes/isp/16276' `# OVH SAS` \
'/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,20115,33363' `# Spectrum` \
'/api/v1/lightning/nodes/isp/701' `# Verizon` \
'/api/v1/lightning/nodes/isp/12876' `# Scaleway` \
'/api/v1/lightning/nodes/isp/33915' `# Ziggo` \
'/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` \
'/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` \
'/api/v1/lightning/nodes/isp/212531', `# UAB Interneto vizija` \
'/api/v1/lightning/nodes/isp/63949', `# Linode` \
'/api/v1/lightning/nodes/isp/51167', `# Contabo GmbH` \
'/api/v1/lightning/nodes/countries' \
'/api/v1/lightning/nodes/rankings' \
'/api/v1/lightning/nodes/rankings/liquidity' \
'/api/v1/lightning/nodes/rankings/connectivity' \
'/api/v1/lightning/nodes/rankings/age' \
warmSlurpURLs=(
'/'
'/mining'
'/lightning'
'/graphs'
'/docs'
'/docs/faq'
'/docs/api/rest'
'/about'
)
warmUnfurlURLs=(
'/'
'/mining'
'/lightning'
)
warmURLs=(
'/api/v1/blocks'
'/api/v1/statistics/2h'
'/api/v1/statistics/24h'
'/api/v1/statistics/1w'
'/api/v1/statistics/1m'
'/api/v1/statistics/3m'
'/api/v1/statistics/6m'
'/api/v1/statistics/1y'
'/api/v1/statistics/2y'
'/api/v1/statistics/3y'
'/api/v1/statistics/4y'
'/api/v1/statistics/all'
'/api/v1/mining/pools/24h'
'/api/v1/mining/pools/3d'
'/api/v1/mining/pools/1w'
'/api/v1/mining/pools/1m'
'/api/v1/mining/pools/3m'
'/api/v1/mining/pools/6m'
'/api/v1/mining/pools/1y'
'/api/v1/mining/pools/2y'
'/api/v1/mining/pools/3y'
'/api/v1/mining/pools/all'
'/api/v1/mining/hashrate/3m'
'/api/v1/mining/hashrate/6m'
'/api/v1/mining/hashrate/1y'
'/api/v1/mining/hashrate/2y'
'/api/v1/mining/hashrate/3y'
'/api/v1/mining/hashrate/all'
'/api/v1/mining/hashrate/pools/3m'
'/api/v1/mining/hashrate/pools/6m'
'/api/v1/mining/hashrate/pools/1y'
'/api/v1/mining/hashrate/pools/2y'
'/api/v1/mining/hashrate/pools/3y'
'/api/v1/mining/hashrate/pools/all'
'/api/v1/mining/reward-stats/144'
'/api/v1/mining/blocks/fees/24h'
'/api/v1/mining/blocks/fees/3d'
'/api/v1/mining/blocks/fees/1w'
'/api/v1/mining/blocks/fees/1m'
'/api/v1/mining/blocks/fees/3m'
'/api/v1/mining/blocks/fees/6m'
'/api/v1/mining/blocks/fees/1y'
'/api/v1/mining/blocks/fees/2y'
'/api/v1/mining/blocks/fees/3y'
'/api/v1/mining/blocks/fees/all'
'/api/v1/mining/blocks/rewards/24h'
'/api/v1/mining/blocks/rewards/3d'
'/api/v1/mining/blocks/rewards/1w'
'/api/v1/mining/blocks/rewards/1m'
'/api/v1/mining/blocks/rewards/3m'
'/api/v1/mining/blocks/rewards/6m'
'/api/v1/mining/blocks/rewards/1y'
'/api/v1/mining/blocks/rewards/2y'
'/api/v1/mining/blocks/rewards/3y'
'/api/v1/mining/blocks/rewards/all'
'/api/v1/mining/blocks/fee-rates/24h'
'/api/v1/mining/blocks/fee-rates/3d'
'/api/v1/mining/blocks/fee-rates/1w'
'/api/v1/mining/blocks/fee-rates/1m'
'/api/v1/mining/blocks/fee-rates/3m'
'/api/v1/mining/blocks/fee-rates/6m'
'/api/v1/mining/blocks/fee-rates/1y'
'/api/v1/mining/blocks/fee-rates/2y'
'/api/v1/mining/blocks/fee-rates/3y'
'/api/v1/mining/blocks/fee-rates/all'
'/api/v1/mining/difficulty-adjustments/24h'
'/api/v1/mining/difficulty-adjustments/3d'
'/api/v1/mining/difficulty-adjustments/1w'
'/api/v1/mining/difficulty-adjustments/1m'
'/api/v1/mining/difficulty-adjustments/3m'
'/api/v1/mining/difficulty-adjustments/6m'
'/api/v1/mining/difficulty-adjustments/1y'
'/api/v1/mining/difficulty-adjustments/2y'
'/api/v1/mining/difficulty-adjustments/3y'
'/api/v1/mining/difficulty-adjustments/all'
'/api/v1/lightning/channels-geo?style=widget'
'/api/v1/lightning/channels-geo?style=graph'
'/api/v1/lightning/statistics/latest'
'/api/v1/lightning/statistics/1m'
'/api/v1/lightning/statistics/3m'
'/api/v1/lightning/statistics/6m'
'/api/v1/lightning/statistics/1y'
'/api/v1/lightning/statistics/2y'
'/api/v1/lightning/statistics/3y'
'/api/v1/lightning/statistics/all'
'/api/v1/lightning/nodes/isp-ranking'
'/api/v1/lightning/nodes/isp/15169,396982' `# Google`
'/api/v1/lightning/nodes/isp/14618,16509' `# Amazon`
'/api/v1/lightning/nodes/isp/39572' `# DataWeb`
'/api/v1/lightning/nodes/isp/14061' `# Digital Ocean`
'/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner`
'/api/v1/lightning/nodes/isp/394745' `# LunaNode`
'/api/v1/lightning/nodes/isp/45102' `# Alibaba`
'/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany`
'/api/v1/lightning/nodes/isp/7922' `# Comcast Cable`
'/api/v1/lightning/nodes/isp/34197' `# SHRD SARL`
'/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU`
'/api/v1/lightning/nodes/isp/16276' `# OVH SAS`
'/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,2$delay,33363' `# Spectrum`
'/api/v1/lightning/nodes/isp/701' `# Verizon`
'/api/v1/lightning/nodes/isp/12876' `# Scaleway`
'/api/v1/lightning/nodes/isp/33915' `# Ziggo`
'/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG`
'/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure`
'/api/v1/lightning/nodes/isp/212531' `# UAB Interneto vizija`
'/api/v1/lightning/nodes/isp/63949' `# Linode`
'/api/v1/lightning/nodes/isp/51167' `# Contabo GmbH`
'/api/v1/lightning/nodes/countries'
'/api/v1/lightning/nodes/rankings'
'/api/v1/lightning/nodes/rankings/liquidity'
'/api/v1/lightning/nodes/rankings/connectivity'
'/api/v1/lightning/nodes/rankings/age'
)
echo "waiting for mempool backend to start..."
sleep 60
while true
do
echo "starting warm cache cycle..."
for url in $warmSlurpURLs
do
warmSlurp "https://${hostname}${url}"
sleep $delay # delay between queries to not DoS unfurler
done
for url in $warmUnfurlURLs
do
warmUnfurl "https://${hostname}${url}"
sleep $delay # delay between queries to not DoS unfurler
done
for url in $warmURLs
do
warm "https://${hostname}${url}"
sleep 0.25 # wait 250ms between queries to not DoS mariadb
sleep $delay # delay between queries to not DoS mariadb
done
for slug in $slugs
do
warm "https://${hostname}/api/v1/mining/pool/${slug}"
sleep 0.25 # wait 250ms between queries to not DoS mariadb
sleep $delay # delay between queries to not DoS mariadb
warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate"
sleep 0.25 # wait 250ms between queries to not DoS mariadb
sleep $delay # delay between queries to not DoS mariadb
warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks"
sleep 0.25 # wait 250ms between queries to not DoS mariadb
sleep $delay # delay between queries to not DoS mariadb
done
sleep 1
done

8
production/nginx/http-proxy-cache.conf
View file

@ -1,5 +1,7 @@
# proxy cache
proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=100m;
proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=100m;
proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=100m;
proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=600s max_size=200m;
types_hash_max_size 2048;

4
production/nginx/nginx.conf
View file

@ -23,8 +23,8 @@ http {
include mempool/production/nginx/http-language.conf;
# match preview/unfurl bot user-agents
map $http_user_agent $unfurlbot {
default 0;
map $http_user_agent $unfurlprefix {
default "";
}
# mempool configuration

55
production/nginx/server-common.conf
View file

@ -48,8 +48,8 @@ add_header Vary Cookie;
# for exact / requests, redirect based on $lang
# cache redirect for 5 minutes
location = / {
if ($unfurlbot) {
proxy_pass $mempoolSpaceUnfurler;
if ($unfurlprefix != '') {
rewrite ^(.*)$ $unfurlprefix$1 last;
}
if ($lang != '') {
return 302 $scheme://$host/$lang/;
@ -65,15 +65,15 @@ location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ {
}
# cache everything else for 5 minutes
location ~ ^/([a-z][a-z])$ {
if ($unfurlbot) {
proxy_pass $mempoolSpaceUnfurler;
if ($unfurlprefix != '') {
rewrite ^(.*)$ $unfurlprefix$1 last;
}
try_files $uri /$1/index.html /en-US/index.html =404;
expires 5m;
}
location ~ ^/([a-z][a-z])/ {
if ($unfurlbot) {
proxy_pass $mempoolSpaceUnfurler;
if ($unfurlprefix != '') {
rewrite ^(.*)$ $unfurlprefix$1 last;
}
try_files $uri /$1/index.html /en-US/index.html =404;
expires 5m;
@ -104,16 +104,53 @@ location /preview {
}
# unfurl renderer
location ^~ /render {
proxy_pass $mempoolSpaceUnfurler;
try_files /dev/null @mempool-space-unfurler;
expires 10m;
}
# unfurl handler
location /unfurl/ {
try_files /dev/null @mempool-space-unfurler;
}
location /slurp/ {
try_files /dev/null @mempool-space-slurper;
}
# catch-all for all URLs i.e. /address/foo /tx/foo /block/000
# cache 5 minutes since they change frequently
location / {
if ($unfurlbot) {
proxy_pass $mempoolSpaceUnfurler;
if ($unfurlprefix != '') {
rewrite ^(.*)$ $unfurlprefix$1 last;
}
try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404;
expires 5m;
}
location @mempool-space-unfurler {
proxy_pass $mempoolSpaceUnfurler;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_cache_background_update on;
proxy_cache_use_stale updating;
proxy_cache unfurler;
proxy_cache_valid 200 10m;
proxy_redirect off;
}
location @mempool-space-slurper {
proxy_pass $mempoolSpaceUnfurler;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_cache_background_update on;
proxy_cache_use_stale updating;
proxy_cache slurper;
proxy_cache_valid 200 10m;
proxy_redirect off;
}

26
unfurler/src/index.ts
View file

@ -109,11 +109,15 @@ class Server {
setUpRoutes() {
if (puppeteerEnabled) {
this.app.get('/unfurl/render*', async (req, res) => { return this.renderPreview(req, res) })
this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
} else {
this.app.get('/unfurl/render*', async (req, res) => { return this.renderDisabled(req, res) })
this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) })
}
this.app.get('*', (req, res) => { return this.renderHTML(req, res) })
this.app.get('/unfurl*', (req, res) => { return this.renderHTML(req, res, true) })
this.app.get('/slurp*', (req, res) => { return this.renderHTML(req, res, false) })
this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) })
}
async clusterTask({ page, data: { url, path, action } }) {
@ -240,7 +244,7 @@ class Server {
}
}
async renderHTML(req, res) {
async renderHTML(req, res, unfurl: boolean = false) {
// drop requests for static files
const rawPath = req.params[0];
const match = rawPath.match(/\.[\w]+$/);
@ -250,25 +254,25 @@ class Server {
|| rawPath.startsWith('/api/v1/translators/images')
|| rawPath.startsWith('/resources/profile')
) {
if (isSearchCrawler(req.headers['user-agent'])) {
if (unfurl) {
res.status(404).send();
return;
} else {
if (this.secureHost) {
https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
} else {
http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
}
return;
} else {
res.status(404).send();
return;
}
}
let result = '';
try {
if (isSearchCrawler(req.headers['user-agent'])) {
result = await this.renderSEOPage(rawPath);
} else {
if (unfurl) {
result = await this.renderUnfurlMeta(rawPath);
} else {
result = await this.renderSEOPage(rawPath);
}
if (result && result.length) {
if (result === '404') {
@ -348,7 +352,3 @@ function capitalize(str) {
return str;
}
}
function isSearchCrawler(useragent: string): boolean {
return /googlebot|applebot|bingbot/i.test(useragent);
}