mirror of
https://github.com/mempool/mempool.git
synced 2025-02-24 14:50:52 +01:00
Merge pull request #4279 from mempool/knorrium/hardened_sync_assets
Fixes to sync-assets
This commit is contained in:
softsimon
GitHub
commit
9cf2db77b4
2 changed files with 110 additions and 44 deletions
|
|
@ -35,7 +35,7 @@
|
|||
"start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
|
||||
"start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed",
|
||||
"build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
|
||||
"sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources'",
|
||||
"sync-assets": "rsync -av ./src/resources ./dist/mempool/browser && node sync-assets.js 'dist/mempool/browser/resources/'",
|
||||
"sync-assets-dev": "node sync-assets.js 'src/resources/'",
|
||||
"generate-config": "node generate-config.js",
|
||||
"build-mempool.js": "npm run build-mempool-js && npm run build-mempool-liquid-js && npm run build-mempool-bisq-js",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,19 @@
|
|||
var https = require('https');
|
||||
var fs = require('fs');
|
||||
var crypto = require('crypto');
|
||||
var path = require('node:path');
|
||||
const LOG_TAG = '[sync-assets]';
|
||||
let verbose = false;
|
||||
|
||||
if (parseInt(process.env.SKIP_SYNC) === 1) {
|
||||
console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (parseInt(process.env.VERBOSE) === 1) {
|
||||
console.log(`${LOG_TAG} VERBOSE is set, logs will be more verbose`);
|
||||
verbose = true;
|
||||
}
|
||||
|
||||
const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
|
||||
let configContent = {};
|
||||
|
|
@ -8,6 +21,13 @@ let configContent = {};
|
|||
var PATH;
|
||||
if (process.argv[2]) {
|
||||
PATH = process.argv[2];
|
||||
PATH += PATH.endsWith("/") ? "" : "/"
|
||||
PATH = path.normalize(PATH);
|
||||
console.log(`[sync-assets] using PATH ${PATH}`);
|
||||
if (!fs.existsSync(PATH)){
|
||||
console.log(`${LOG_TAG} ${PATH} does not exist, creating`);
|
||||
fs.mkdirSync(PATH, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (!PATH) {
|
||||
|
|
@ -17,12 +37,12 @@ if (!PATH) {
|
|||
try {
|
||||
const rawConfig = fs.readFileSync(CONFIG_FILE_NAME);
|
||||
configContent = JSON.parse(rawConfig);
|
||||
console.log(`${CONFIG_FILE_NAME} file found, using provided config`);
|
||||
console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file found, using provided config`);
|
||||
} catch (e) {
|
||||
if (e.code !== 'ENOENT') {
|
||||
throw new Error(e);
|
||||
} else {
|
||||
console.log(`${CONFIG_FILE_NAME} file not found, using default config`);
|
||||
console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file not found, using default config`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -37,6 +57,11 @@ function download(filename, url) {
|
|||
})
|
||||
.on('error', function(e) {
|
||||
throw new Error(e);
|
||||
})
|
||||
.on('finish', () => {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Finished downloading ${url} to ${filename}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -44,12 +69,18 @@ function getLocalHash(filePath) {
|
|||
const size = fs.statSync(filePath);
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]);
|
||||
return crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
|
||||
const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
|
||||
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \tgetLocalHash ${filePath} ${hash}`);
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
function downloadMiningPoolLogos$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log('Checking if mining pool logos needs downloading or updating...');
|
||||
console.log(`${LOG_TAG} Checking if mining pool logos needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mining-pool-logos/contents/',
|
||||
|
|
@ -58,7 +89,7 @@ function downloadMiningPoolLogos$() {
|
|||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log('Downloading the mining pool logos with authentication');
|
||||
console.log(`${LOG_TAG} Downloading the mining pool logos with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
}
|
||||
|
|
@ -79,21 +110,29 @@ function downloadMiningPoolLogos$() {
|
|||
}
|
||||
let downloadedCount = 0;
|
||||
for (const poolLogo of poolLogos) {
|
||||
const filePath = `${PATH}/mining-pools/${poolLogo.name}`;
|
||||
const filePath = PATH + `mining-pools/${poolLogo.name}`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
const localHash = getLocalHash(filePath);
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Remote ${poolLogo.name} logo hash ${poolLogo.sha}`);
|
||||
console.log(`${LOG_TAG} \tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`);
|
||||
}
|
||||
if (localHash !== poolLogo.sha) {
|
||||
console.log(`${poolLogo.name} is different on the remote, downloading...`);
|
||||
console.log(`${LOG_TAG} \t\t${poolLogo.name} is different on the remote, downloading...`);
|
||||
download(filePath, poolLogo.download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
} else {
|
||||
console.log(`${poolLogo.name} is missing, downloading...`);
|
||||
console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`);
|
||||
const miningPoolsDir = PATH + `mining-pools/`;
|
||||
if (!fs.existsSync(miningPoolsDir)){
|
||||
fs.mkdirSync(miningPoolsDir, { recursive: true });
|
||||
}
|
||||
download(filePath, poolLogo.download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
}
|
||||
console.log(`Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
|
||||
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
|
|
@ -109,7 +148,7 @@ function downloadMiningPoolLogos$() {
|
|||
|
||||
function downloadPromoVideoSubtiles$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log('Checking if promo video subtitles needs downloading or updating...');
|
||||
console.log(`${LOG_TAG} Checking if promo video subtitles needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mempool-promo/contents/subtitles',
|
||||
|
|
@ -118,7 +157,7 @@ function downloadPromoVideoSubtiles$() {
|
|||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log('Downloading the promo video subtitles with authentication');
|
||||
console.log(`${LOG_TAG} Downloading the promo video subtitles with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
}
|
||||
|
|
@ -140,21 +179,30 @@ function downloadPromoVideoSubtiles$() {
|
|||
}
|
||||
let downloadedCount = 0;
|
||||
for (const language of videoLanguages) {
|
||||
const filePath = `${PATH}/promo-video/${language.name}`;
|
||||
const filePath = PATH + `promo-video/${language.name}`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`);
|
||||
}
|
||||
const localHash = getLocalHash(filePath);
|
||||
|
||||
if (localHash !== language.sha) {
|
||||
console.log(`${language.name} is different on the remote, updating`);
|
||||
console.log(`${LOG_TAG} ${language.name} is different on the remote, updating`);
|
||||
download(filePath, language.download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
} else {
|
||||
console.log(`${language.name} is missing, downloading`);
|
||||
console.log(`${LOG_TAG} ${language.name} is missing, downloading`);
|
||||
const promoVideosDir = PATH + `promo-video/`;
|
||||
if (!fs.existsSync(promoVideosDir)){
|
||||
fs.mkdirSync(promoVideosDir, { recursive: true });
|
||||
}
|
||||
|
||||
download(filePath, language.download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
}
|
||||
console.log(`Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
|
||||
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
|
|
@ -170,7 +218,7 @@ function downloadPromoVideoSubtiles$() {
|
|||
|
||||
function downloadPromoVideo$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log('Checking if promo video needs downloading or updating...');
|
||||
console.log(`${LOG_TAG} Checking if promo video needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mempool-promo/contents',
|
||||
|
|
@ -179,7 +227,7 @@ function downloadPromoVideo$() {
|
|||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log('Downloading the promo videos with authentication');
|
||||
console.log(`${LOG_TAG} Downloading the promo video with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
}
|
||||
|
|
@ -202,18 +250,19 @@ function downloadPromoVideo$() {
|
|||
if (item.name !== 'promo.mp4') {
|
||||
continue;
|
||||
}
|
||||
const filePath = `${PATH}/promo-video/mempool-promo.mp4`;
|
||||
const filePath = PATH + `promo-video/mempool-promo.mp4`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
const localHash = getLocalHash(filePath);
|
||||
|
||||
if (localHash !== item.sha) {
|
||||
console.log(`mempool-promo.mp4 is different on the remote, updating`);
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`);
|
||||
download(filePath, item.download_url);
|
||||
console.log('mempool-promo.mp4 downloaded.');
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`);
|
||||
} else {
|
||||
console.log(`mempool-promo.mp4 is already up to date. Skipping.`);
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is already up to date. Skipping.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`mempool-promo.mp4 is missing, downloading`);
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`);
|
||||
download(filePath, item.download_url);
|
||||
}
|
||||
}
|
||||
|
|
@ -232,30 +281,47 @@ function downloadPromoVideo$() {
|
|||
}
|
||||
|
||||
|
||||
|
||||
let assetsJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.json';
|
||||
let assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/mempool/asset_registry_db/master/index.minimal.json';
|
||||
|
||||
if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
|
||||
assetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json';
|
||||
assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json';
|
||||
const assetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json';
|
||||
const assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json';
|
||||
const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
|
||||
const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';
|
||||
|
||||
console.log(`${LOG_TAG} Downloading assets`);
|
||||
download(PATH + 'assets.json', assetsJsonUrl);
|
||||
|
||||
console.log(`${LOG_TAG} Downloading assets minimal`);
|
||||
download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl);
|
||||
|
||||
console.log(`${LOG_TAG} Downloading testnet assets`);
|
||||
download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl);
|
||||
|
||||
console.log(`${LOG_TAG} Downloading testnet assets minimal`);
|
||||
download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (${configContent.BASE_MODULE}), skipping downloading assets`);
|
||||
}
|
||||
}
|
||||
|
||||
const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
|
||||
const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';
|
||||
|
||||
console.log('Downloading assets');
|
||||
download(PATH + 'assets.json', assetsJsonUrl);
|
||||
console.log('Downloading assets minimal');
|
||||
download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl);
|
||||
console.log('Downloading testnet assets');
|
||||
download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl);
|
||||
console.log('Downloading testnet assets minimal');
|
||||
download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl);
|
||||
|
||||
downloadMiningPoolLogos$()
|
||||
.then(() => downloadPromoVideoSubtiles$())
|
||||
.then(() => downloadPromoVideo$())
|
||||
(() => {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Downloading mining pool logos`);
|
||||
}
|
||||
downloadMiningPoolLogos$()
|
||||
.then(() => {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Downloading promo video subtitles`);
|
||||
}
|
||||
downloadPromoVideoSubtiles$();
|
||||
})
|
||||
.then(() => {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Downloading promo video`);
|
||||
}
|
||||
downloadPromoVideo$();
|
||||
})
|
||||
.catch((error) => {
|
||||
throw new Error(error);
|
||||
});
|
||||
})();
|
||||
Loading…
Add table
Reference in a new issue