mirror of
https://github.com/mempool/mempool.git
synced 2025-01-18 21:32:55 +01:00
Add DRY_RUN and MEMPOOL_CDN options
This commit is contained in:
Felipe Knorr Kuhn
parent
a15729c38f
commit
933bc47b44
@ -4,6 +4,8 @@ var crypto = require('crypto');
|
||||
var path = require('node:path');
|
||||
const LOG_TAG = '[sync-assets]';
|
||||
let verbose = false;
|
||||
let MEMPOOL_CDN = false;
|
||||
let DRY_RUN = false;
|
||||
|
||||
if (parseInt(process.env.SKIP_SYNC) === 1) {
|
||||
console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`);
|
||||
@ -15,6 +17,18 @@ if (parseInt(process.env.VERBOSE) === 1) {
|
||||
verbose = true;
|
||||
}
|
||||
|
||||
if (parseInt(process.env.MEMPOOL_CDN) === 1) {
|
||||
console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`);
|
||||
MEMPOOL_CDN = true;
|
||||
}
|
||||
|
||||
if (parseInt(process.env.DRY_RUN) === 1) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`);
|
||||
DRY_RUN = true;
|
||||
}
|
||||
|
||||
const githubSecret = process.env.GITHUB_TOKEN;
|
||||
|
||||
const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
|
||||
let configContent = {};
|
||||
|
||||
@ -46,8 +60,6 @@ try {
|
||||
}
|
||||
}
|
||||
|
||||
const githubSecret = process.env.GITHUB_TOKEN;
|
||||
|
||||
function download(filename, url) {
|
||||
https.get(url, (response) => {
|
||||
if (response.statusCode < 200 || response.statusCode > 299) {
|
||||
@ -60,7 +72,7 @@ function download(filename, url) {
|
||||
})
|
||||
.on('finish', () => {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Finished downloading ${url} to ${filename}`);
|
||||
console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filename}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
@ -72,7 +84,7 @@ function getLocalHash(filePath) {
|
||||
const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex');
|
||||
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \tgetLocalHash ${filePath} ${hash}`);
|
||||
console.log(`${LOG_TAG} \t\tgetLocalHash ${filePath} ${hash}`);
|
||||
}
|
||||
|
||||
return hash;
|
||||
@ -80,7 +92,7 @@ function getLocalHash(filePath) {
|
||||
|
||||
function downloadMiningPoolLogos$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`${LOG_TAG} Checking if mining pool logos needs downloading or updating...`);
|
||||
console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mining-pool-logos/contents/',
|
||||
@ -110,29 +122,54 @@ function downloadMiningPoolLogos$() {
|
||||
}
|
||||
let downloadedCount = 0;
|
||||
for (const poolLogo of poolLogos) {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Processing ${poolLogo.name}`);
|
||||
}
|
||||
const filePath = `${PATH}/mining-pools/${poolLogo.name}`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
const localHash = getLocalHash(filePath);
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Remote ${poolLogo.name} logo hash ${poolLogo.sha}`);
|
||||
console.log(`${LOG_TAG} \tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`);
|
||||
console.log(`${LOG_TAG} \t\tremote ${poolLogo.name} logo hash ${poolLogo.sha}`);
|
||||
console.log(`${LOG_TAG} \t\t\tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`);
|
||||
}
|
||||
if (localHash !== poolLogo.sha) {
|
||||
console.log(`${LOG_TAG} \t\t${poolLogo.name} is different on the remote, downloading...`);
|
||||
download(filePath, poolLogo.download_url);
|
||||
downloadedCount++;
|
||||
console.log(`${LOG_TAG} \t\t\t\t${poolLogo.name} is different on the remote, downloading...`);
|
||||
let download_url = poolLogo.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${poolLogo.name} but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \t\tDownloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${poolLogo.name} is already up to date. Skipping.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`);
|
||||
console.log(`${LOG_TAG} \t\t${poolLogo.name} is missing, downloading...`);
|
||||
const miningPoolsDir = `${PATH}/mining-pools/`;
|
||||
if (!fs.existsSync(miningPoolsDir)){
|
||||
fs.mkdirSync(miningPoolsDir, { recursive: true });
|
||||
}
|
||||
download(filePath, poolLogo.download_url);
|
||||
downloadedCount++;
|
||||
let download_url = poolLogo.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading ${poolLogo.name} but it should`);
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \tDownloading ${download_url} to ${filePath}`);
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
|
||||
console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
|
||||
@ -148,7 +185,7 @@ function downloadMiningPoolLogos$() {
|
||||
|
||||
function downloadPromoVideoSubtiles$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`${LOG_TAG} Checking if promo video subtitles needs downloading or updating...`);
|
||||
console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mempool-promo/contents/subtitles',
|
||||
@ -157,7 +194,7 @@ function downloadPromoVideoSubtiles$() {
|
||||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log(`${LOG_TAG} Downloading the promo video subtitles with authentication`);
|
||||
console.log(`${LOG_TAG} \tDownloading the promo video subtitles with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
}
|
||||
@ -179,27 +216,53 @@ function downloadPromoVideoSubtiles$() {
|
||||
}
|
||||
let downloadedCount = 0;
|
||||
for (const language of videoLanguages) {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} Processing ${language.name}`);
|
||||
}
|
||||
const filePath = `${PATH}/promo-video/${language.name}`;
|
||||
if (fs.existsSync(filePath)) {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`);
|
||||
console.log(`${LOG_TAG} \t${language.name} remote promo video hash ${language.sha}`);
|
||||
}
|
||||
const localHash = getLocalHash(filePath);
|
||||
|
||||
if (localHash !== language.sha) {
|
||||
console.log(`${LOG_TAG} ${language.name} is different on the remote, updating`);
|
||||
download(filePath, language.download_url);
|
||||
downloadedCount++;
|
||||
console.log(`${LOG_TAG} \t\t${language.name} is different on the remote, updating`);
|
||||
let download_url = language.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${language.name} but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} \t\tdownloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \t\t${language.name} is already up to date. Skipping.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} ${language.name} is missing, downloading`);
|
||||
console.log(`${LOG_TAG} \t\t${language.name} is missing, downloading`);
|
||||
const promoVideosDir = `${PATH}/promo-video/`;
|
||||
if (!fs.existsSync(promoVideosDir)){
|
||||
fs.mkdirSync(promoVideosDir, { recursive: true });
|
||||
}
|
||||
|
||||
download(filePath, language.download_url);
|
||||
downloadedCount++;
|
||||
let download_url = language.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = downloadownload_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} \tDRY_RUN is set, not downloading ${language.name} but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
downloadedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
|
||||
@ -218,7 +281,7 @@ function downloadPromoVideoSubtiles$() {
|
||||
|
||||
function downloadPromoVideo$() {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`${LOG_TAG} Checking if promo video needs downloading or updating...`);
|
||||
console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`);
|
||||
const options = {
|
||||
host: 'api.github.com',
|
||||
path: '/repos/mempool/mempool-promo/contents',
|
||||
@ -227,7 +290,7 @@ function downloadPromoVideo$() {
|
||||
};
|
||||
|
||||
if (githubSecret) {
|
||||
console.log(`${LOG_TAG} Downloading the promo video with authentication`);
|
||||
console.log(`${LOG_TAG} \tDownloading the promo video with authentication`);
|
||||
options.headers['authorization'] = `Bearer ${githubSecret}`;
|
||||
options.headers['X-GitHub-Api-Version'] = '2022-11-28';
|
||||
}
|
||||
@ -256,14 +319,36 @@ function downloadPromoVideo$() {
|
||||
|
||||
if (localHash !== item.sha) {
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`);
|
||||
download(filePath, item.download_url);
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`);
|
||||
let download_url = item.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is already up to date. Skipping.`);
|
||||
console.log(`${LOG_TAG} \t\tmempool-promo.mp4 is already up to date. Skipping.`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`);
|
||||
download(filePath, item.download_url);
|
||||
let download_url = item.download_url;
|
||||
if (MEMPOOL_CDN) {
|
||||
download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
|
||||
}
|
||||
if (DRY_RUN) {
|
||||
console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
|
||||
}
|
||||
download(filePath, download_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
resolve();
|
||||
@ -300,7 +385,7 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
|
||||
download(`${PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl);
|
||||
} else {
|
||||
if (verbose) {
|
||||
console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (${configContent.BASE_MODULE}), skipping downloading assets`);
|
||||
console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user