From 8ec61dd603cb52147f539e5f2c0b148a1a5e78e8 Mon Sep 17 00:00:00 2001 From: wiz Date: Sun, 28 Aug 2022 13:04:40 +0200 Subject: [PATCH] Update ops scripts for unfurler and cache warmer --- production/mempool-build-all | 18 +++++++ production/mempool-kill-all | 8 +++- production/mempool-start-all | 22 +++++++++ production/nginx-cache-warmer | 14 ++++-- production/unfurl-build | 62 ------------------------- production/unfurl-kill | 2 - production/unfurl-start | 6 --- production/unfurler-config.liquid.json | 17 +++++++ production/unfurler-config.mainnet.json | 17 +++++++ unfurler/package.json | 2 +- unfurler/puppeteer.config.json | 2 +- 11 files changed, 93 insertions(+), 77 deletions(-) delete mode 100755 production/unfurl-build delete mode 100755 production/unfurl-kill delete mode 100755 production/unfurl-start create mode 100644 production/unfurler-config.liquid.json create mode 100644 production/unfurler-config.mainnet.json diff --git a/production/mempool-build-all b/production/mempool-build-all index c0e9a2c2a..4d0e51d24 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -63,6 +63,19 @@ build_frontend() npm run build || exit 1 } +build_unfurler() +{ + local site="$1" + echo "[*] Building unfurler for ${site}" + [ -z "${HASH}" ] && exit 1 + cd "$HOME/${site}/unfurler" || exit 1 + if [ ! -e "config.json" ];then + cp "${HOME}/mempool/production/unfurler-config.${site}.json" "config.json" + fi + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install || exit 1 + npm run build || exit 1 +} + build_backend() { local site="$1" @@ -128,6 +141,11 @@ for repo in $backend_repos;do update_repo "${repo}" done +# build unfurlers +for repo in mainnet liquid;do + build_unfurler "${repo}" +done + # build backends for repo in $backend_repos;do build_backend "${repo}" diff --git a/production/mempool-kill-all b/production/mempool-kill-all index ae48552c2..b75969155 100755 --- a/production/mempool-kill-all +++ b/production/mempool-kill-all @@ -1,2 +1,8 @@ #!/usr/bin/env zsh -killall sh node +killall sh +killall node +killall chrome +killall xinit +for pid in `ps uaxww|grep warmer|grep zsh|awk '{print $2}'`;do + kill $pid +done diff --git a/production/mempool-start-all b/production/mempool-start-all index 94766d5ce..a6fdfa589 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -2,7 +2,29 @@ export NVM_DIR="$HOME/.nvm" source "$NVM_DIR/nvm.sh" +# start all mempool backends that exist for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do cd "${HOME}/${site}/backend/" && \ + echo "starting mempool backend: ${site}" && \ screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done' done + +# only start unfurler if GPU present +if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then + export DISPLAY=:0 + screen -dmS x startx + sleep 3 + for site in mainnet liquid;do + cd "$HOME/${site}/unfurler" && \ + echo "starting mempool unfurler: ${site}" && \ + screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done' + done +fi + +# start nginx warm cacher +for site in mainnet;do + echo "starting mempool cache warmer: ${site}" + screen -dmS "warmer-${site}" $HOME/mempool/production/nginx-cache-warmer +done + +exit 0 diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index a4ece6e0b..3c0fd8ef1 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -2,6 +2,12 @@ hostname=$(hostname) slugs=(`curl -sSL https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json | jq -r '.slugs[]'`) +warm() +{ + echo "$1" + curl -i -s "$1" | head -1 +} + while true do for url in / \ '/api/v1/blocks' \ @@ -81,14 +87,14 @@ do for url in / \ '/api/v1/lightning/channels-geo?style=graph' \ do - curl -s "https://${hostname}${url}" >/dev/null + warm "https://${hostname}${url}" done for slug in $slugs do - curl -s "https://${hostname}/api/v1/mining/pool/${slug}" >/dev/null - curl -s "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" >/dev/null - curl -s "https://${hostname}/api/v1/mining/pool/${slug}/blocks" >/dev/null + warm "https://${hostname}/api/v1/mining/pool/${slug}" + warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" + warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks" done sleep 10 diff --git a/production/unfurl-build b/production/unfurl-build deleted file mode 100755 index 5b838e0ae..000000000 --- a/production/unfurl-build +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env zsh -PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin -HOSTNAME=$(hostname) -LOCATION=$(hostname|cut -d . -f2) -LOCKFILE="${HOME}/lock" -REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!') - -if [ -f "${LOCKFILE}" ];then - echo "upgrade already running? check lockfile ${LOCKFILE}" - exit 1 -fi - -# on exit, remove lockfile but preserve exit code -trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT - -# create lockfile -touch "${LOCKFILE}" - -# notify logged in users -echo "Upgrading unfurler to ${REF}" | wall - -update_repo() -{ - echo "[*] Upgrading unfurler to ${REF}" - cd "$HOME/unfurl/unfurler" || exit 1 - - git fetch origin || exit 1 - for remote in origin;do - git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1 - git fetch "${remote}" || exit 1 - done - - if [ $(git tag -l "${REF}") ];then - git reset --hard "tags/${REF}" || exit 1 - elif [ $(git branch -r -l "origin/${REF}") ];then - git reset --hard "origin/${REF}" || exit 1 - else - git reset --hard "${REF}" || exit 1 - fi - export HASH=$(git rev-parse HEAD) -} - -build_backend() -{ - echo "[*] Building backend for unfurler" - [ -z "${HASH}" ] && exit 1 - cd "$HOME/unfurl/unfurler" || exit 1 - if [ ! -e "config.json" ];then - cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json" - fi - npm install || exit 1 - npm run build || exit 1 -} - -update_repo -build_backend - -# notify everyone -echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev -echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}" - -exit 0 diff --git a/production/unfurl-kill b/production/unfurl-kill deleted file mode 100755 index ae48552c2..000000000 --- a/production/unfurl-kill +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env zsh -killall sh node diff --git a/production/unfurl-start b/production/unfurl-start deleted file mode 100755 index 29b5ddf3e..000000000 --- a/production/unfurl-start +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env zsh -export NVM_DIR="$HOME/.nvm" -source "$NVM_DIR/nvm.sh" - -cd "${HOME}/unfurl/unfurler/" && \ -screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done' diff --git a/production/unfurler-config.liquid.json b/production/unfurler-config.liquid.json new file mode 100644 index 000000000..39df7e3fd --- /dev/null +++ b/production/unfurler-config.liquid.json @@ -0,0 +1,17 @@ +{ + "SERVER": { + "HOST": "https://liquid.network", + "HTTP_PORT": 8002 + }, + "MEMPOOL": { + "HTTP_HOST": "https://liquid.network", + "HTTP_PORT": 443, + "NETWORK": "liquid" + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8, + "EXEC_PATH": "/usr/local/bin/chrome", + "MAX_PAGE_AGE": 86400, + "RENDER_TIMEOUT": 3000 + } +} diff --git a/production/unfurler-config.mainnet.json b/production/unfurler-config.mainnet.json new file mode 100644 index 000000000..752cd5706 --- /dev/null +++ b/production/unfurler-config.mainnet.json @@ -0,0 +1,17 @@ +{ + "SERVER": { + "HOST": "https://mempool.space", + "HTTP_PORT": 8001 + }, + "MEMPOOL": { + "HTTP_HOST": "https://mempool.space", + "HTTP_PORT": 443, + "NETWORK": "bitcoin" + }, + "PUPPETEER": { + "CLUSTER_SIZE": 8, + "EXEC_PATH": "/usr/local/bin/chrome", + "MAX_PAGE_AGE": 86400, + "RENDER_TIMEOUT": 3000 + } +} diff --git a/unfurler/package.json b/unfurler/package.json index 2d353bfdf..ca60201b3 100644 --- a/unfurler/package.json +++ b/unfurler/package.json @@ -11,7 +11,7 @@ "tsc": "./node_modules/typescript/bin/tsc", "build": "npm run tsc", "start": "node --max-old-space-size=2048 dist/index.js", - "start-production": "node --max-old-space-size=4096 dist/index.js", + "unfurler": "node --max-old-space-size=4096 dist/index.js", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"" diff --git a/unfurler/puppeteer.config.json b/unfurler/puppeteer.config.json index 3de7b0652..b3a9b7fc4 100644 --- a/unfurler/puppeteer.config.json +++ b/unfurler/puppeteer.config.json @@ -41,6 +41,6 @@ "--use-mock-keychain", "--ignore-gpu-blacklist", "--ignore-gpu-blocklist", - "--use-gl=swiftshader" + "--use-gl=egl" ] }