diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index b15b5e315..2ff642e27 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -4,6 +4,8 @@ var crypto = require('crypto'); var path = require('node:path'); const LOG_TAG = '[sync-assets]'; let verbose = false; +let MEMPOOL_CDN = false; +let DRY_RUN = false; if (parseInt(process.env.SKIP_SYNC) === 1) { console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`); @@ -15,6 +17,18 @@ if (parseInt(process.env.VERBOSE) === 1) { verbose = true; } +if (parseInt(process.env.MEMPOOL_CDN) === 1) { + console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`); + MEMPOOL_CDN = true; +} + +if (parseInt(process.env.DRY_RUN) === 1) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`); + DRY_RUN = true; +} + +const githubSecret = process.env.GITHUB_TOKEN; + const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; let configContent = {}; @@ -46,8 +60,6 @@ try { } } -const githubSecret = process.env.GITHUB_TOKEN; - function download(filename, url) { https.get(url, (response) => { if (response.statusCode < 200 || response.statusCode > 299) { @@ -60,7 +72,7 @@ function download(filename, url) { }) .on('finish', () => { if (verbose) { - console.log(`${LOG_TAG} Finished downloading ${url} to ${filename}`); + console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filename}`); } }); } @@ -72,7 +84,7 @@ function getLocalHash(filePath) { const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); if (verbose) { - console.log(`${LOG_TAG} \tgetLocalHash ${filePath} ${hash}`); + console.log(`${LOG_TAG} \t\tgetLocalHash ${filePath} ${hash}`); } return hash; @@ -80,7 +92,7 @@ function getLocalHash(filePath) { function downloadMiningPoolLogos$() { return new Promise((resolve, reject) => { - console.log(`${LOG_TAG} Checking if mining pool logos needs downloading or updating...`); + console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mining-pool-logos/contents/', @@ -110,29 +122,54 @@ function downloadMiningPoolLogos$() { } let downloadedCount = 0; for (const poolLogo of poolLogos) { + if (verbose) { + console.log(`${LOG_TAG} Processing ${poolLogo.name}`); + } const filePath = `${PATH}/mining-pools/${poolLogo.name}`; if (fs.existsSync(filePath)) { const localHash = getLocalHash(filePath); if (verbose) { - console.log(`${LOG_TAG} Remote ${poolLogo.name} logo hash ${poolLogo.sha}`); - console.log(`${LOG_TAG} \tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`); + console.log(`${LOG_TAG} \t\tremote ${poolLogo.name} logo hash ${poolLogo.sha}`); + console.log(`${LOG_TAG} \t\t\tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`); } if (localHash !== poolLogo.sha) { - console.log(`${LOG_TAG} \t\t${poolLogo.name} is different on the remote, downloading...`); - download(filePath, poolLogo.download_url); - downloadedCount++; + console.log(`${LOG_TAG} \t\t\t\t${poolLogo.name} is different on the remote, downloading...`); + let download_url = poolLogo.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${poolLogo.name} but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} \t\tDownloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + downloadedCount++; + } + } else { + console.log(`${LOG_TAG} \t\t${poolLogo.name} is already up to date. Skipping.`); } } else { - console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`); + console.log(`${LOG_TAG} \t\t${poolLogo.name} is missing, downloading...`); const miningPoolsDir = `${PATH}/mining-pools/`; if (!fs.existsSync(miningPoolsDir)){ fs.mkdirSync(miningPoolsDir, { recursive: true }); } - download(filePath, poolLogo.download_url); - downloadedCount++; + let download_url = poolLogo.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading ${poolLogo.name} but it should`); + } else { + console.log(`${LOG_TAG} \tDownloading ${download_url} to ${filePath}`); + download(filePath, download_url); + downloadedCount++; + } } } - console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); + console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); resolve(); } catch (e) { reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); @@ -148,7 +185,7 @@ function downloadMiningPoolLogos$() { function downloadPromoVideoSubtiles$() { return new Promise((resolve, reject) => { - console.log(`${LOG_TAG} Checking if promo video subtitles needs downloading or updating...`); + console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mempool-promo/contents/subtitles', @@ -157,7 +194,7 @@ function downloadPromoVideoSubtiles$() { }; if (githubSecret) { - console.log(`${LOG_TAG} Downloading the promo video subtitles with authentication`); + console.log(`${LOG_TAG} \tDownloading the promo video subtitles with authentication`); options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['X-GitHub-Api-Version'] = '2022-11-28'; } @@ -179,27 +216,53 @@ function downloadPromoVideoSubtiles$() { } let downloadedCount = 0; for (const language of videoLanguages) { + if (verbose) { + console.log(`${LOG_TAG} Processing ${language.name}`); + } const filePath = `${PATH}/promo-video/${language.name}`; if (fs.existsSync(filePath)) { if (verbose) { - console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`); + console.log(`${LOG_TAG} \t${language.name} remote promo video hash ${language.sha}`); } const localHash = getLocalHash(filePath); - if (localHash !== language.sha) { - console.log(`${LOG_TAG} ${language.name} is different on the remote, updating`); - download(filePath, language.download_url); - downloadedCount++; + console.log(`${LOG_TAG} \t\t${language.name} is different on the remote, updating`); + let download_url = language.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${language.name} but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} \t\tdownloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + downloadedCount++; + } + } else { + console.log(`${LOG_TAG} \t\t${language.name} is already up to date. Skipping.`); } } else { - console.log(`${LOG_TAG} ${language.name} is missing, downloading`); + console.log(`${LOG_TAG} \t\t${language.name} is missing, downloading`); const promoVideosDir = `${PATH}/promo-video/`; if (!fs.existsSync(promoVideosDir)){ fs.mkdirSync(promoVideosDir, { recursive: true }); } - download(filePath, language.download_url); - downloadedCount++; + let download_url = language.download_url; + if (MEMPOOL_CDN) { + download_url = downloadownload_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} \tDRY_RUN is set, not downloading ${language.name} but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + downloadedCount++; + } } } console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); @@ -218,7 +281,7 @@ function downloadPromoVideoSubtiles$() { function downloadPromoVideo$() { return new Promise((resolve, reject) => { - console.log(`${LOG_TAG} Checking if promo video needs downloading or updating...`); + console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mempool-promo/contents', @@ -227,7 +290,7 @@ function downloadPromoVideo$() { }; if (githubSecret) { - console.log(`${LOG_TAG} Downloading the promo video with authentication`); + console.log(`${LOG_TAG} \tDownloading the promo video with authentication`); options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['X-GitHub-Api-Version'] = '2022-11-28'; } @@ -256,14 +319,36 @@ function downloadPromoVideo$() { if (localHash !== item.sha) { console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`); - download(filePath, item.download_url); - console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`); + let download_url = item.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`); + } } else { - console.log(`${LOG_TAG} \tmempool-promo.mp4 is already up to date. Skipping.`); + console.log(`${LOG_TAG} \t\tmempool-promo.mp4 is already up to date. Skipping.`); } } else { console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`); - download(filePath, item.download_url); + let download_url = item.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + } } } resolve(); @@ -300,7 +385,7 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { download(`${PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl); } else { if (verbose) { - console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (${configContent.BASE_MODULE}), skipping downloading assets`); + console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`); } }