From 0565471ecfbfab0d3805042c6087d88f157edb60 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 22 Apr 2022 04:03:08 -0400 Subject: [PATCH 1/2] Wait for external assets file writers to complete --- backend/src/api/common.ts | 2 +- backend/src/api/database-migration.ts | 9 ++- backend/src/api/liquid/icons.ts | 1 - backend/src/index.ts | 6 +- backend/src/sync-assets.ts | 96 +++++++++++++++------------ 5 files changed, 63 insertions(+), 51 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index f9ae196b3..45ef5f576 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -77,7 +77,7 @@ export class Common { }; } - static sleep(ms: number): Promise { + static sleep$(ms: number): Promise { return new Promise((resolve) => { setTimeout(() => { resolve(); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 0e9a18220..45b82a3a5 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -1,8 +1,7 @@ import config from '../config'; import DB from '../database'; import logger from '../logger'; - -const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); +import { Common } from './common'; class DatabaseMigration { private static currentVersion = 17; @@ -25,7 +24,7 @@ class DatabaseMigration { await this.$createMigrationStateTable(); } catch (e) { logger.err('MIGRATIONS: Unable to create `state` table, aborting in 10 seconds. ' + e); - await sleep(10000); + await Common.sleep$(10000); process.exit(-1); } logger.debug('MIGRATIONS: `state` table initialized.'); @@ -36,7 +35,7 @@ class DatabaseMigration { databaseSchemaVersion = await this.$getSchemaVersionFromDatabase(); } catch (e) { logger.err('MIGRATIONS: Unable to get current database migration version, aborting in 10 seconds. ' + e); - await sleep(10000); + await Common.sleep$(10000); process.exit(-1); } @@ -52,7 +51,7 @@ class DatabaseMigration { await this.$createMissingTablesAndIndexes(databaseSchemaVersion); } catch (e) { logger.err('MIGRATIONS: Unable to create required tables, aborting in 10 seconds. ' + e); - await sleep(10000); + await Common.sleep$(10000); process.exit(-1); } diff --git a/backend/src/api/liquid/icons.ts b/backend/src/api/liquid/icons.ts index 1c7c658af..ee08757d0 100644 --- a/backend/src/api/liquid/icons.ts +++ b/backend/src/api/liquid/icons.ts @@ -1,5 +1,4 @@ import * as fs from 'fs'; -import config from '../../config'; import logger from '../../logger'; class Icons { diff --git a/backend/src/index.ts b/backend/src/index.ts index 943448e3a..00ba7cc37 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -85,16 +85,16 @@ class Server { this.setUpWebsocketHandling(); - await syncAssets.syncAssets(); + await syncAssets.syncAssets$(); diskCache.loadMempoolCache(); if (config.DATABASE.ENABLED) { await DB.checkDbConnection(); try { - if (process.env.npm_config_reindex != undefined) { // Re-index requests + if (process.env.npm_config_reindex !== undefined) { // Re-index requests const tables = process.env.npm_config_reindex.split(','); logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`); - await Common.sleep(5000); + await Common.sleep$(5000); await databaseMigration.$truncateIndexedData(tables); } await databaseMigration.$initializeOrMigrateDatabase(); diff --git a/backend/src/sync-assets.ts b/backend/src/sync-assets.ts index d7644e24b..032e71ee2 100644 --- a/backend/src/sync-assets.ts +++ b/backend/src/sync-assets.ts @@ -9,55 +9,69 @@ const PATH = './'; class SyncAssets { constructor() { } - public async syncAssets() { + public async syncAssets$() { for (const url of config.MEMPOOL.EXTERNAL_ASSETS) { - await this.downloadFile(url); + try { + await this.downloadFile$(url); + } catch (e) { + throw new Error(`Failed to download external asset. ` + (e instanceof Error ? e.message : e)); + } } } - private async downloadFile(url: string) { - const fileName = url.split('/').slice(-1)[0]; + private async downloadFile$(url: string) { + return new Promise((resolve, reject) => { + const fileName = url.split('/').slice(-1)[0]; - try { - if (config.SOCKS5PROXY.ENABLED) { - let socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - host: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; + try { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + host: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + const agent = new SocksProxyAgent(socksOptions); + + logger.info(`Downloading external asset ${fileName} over the Tor network...`); + return axios.get(url, { + httpAgent: agent, + httpsAgent: agent, + responseType: 'stream', + timeout: 30000 + }).then(function (response) { + const writer = fs.createWriteStream(PATH + fileName); + writer.on('finish', () => { + logger.info(`External asset ${fileName} saved to ${PATH + fileName}`); + resolve(0); + }); + response.data.pipe(writer); + }); + } else { + logger.info(`Downloading external asset ${fileName} over clearnet...`); + return axios.get(url, { + responseType: 'stream', + timeout: 30000 + }).then(function (response) { + const writer = fs.createWriteStream(PATH + fileName); + writer.on('finish', () => { + logger.info(`External asset ${fileName} saved to ${PATH + fileName}`); + resolve(0); + }); + response.data.pipe(writer); + }); } - - const agent = new SocksProxyAgent(socksOptions); - - logger.info(`Downloading external asset ${fileName} over the Tor network...`); - await axios.get(url, { - httpAgent: agent, - httpsAgent: agent, - responseType: 'stream', - timeout: 30000 - }).then(function (response) { - response.data.pipe(fs.createWriteStream(PATH + fileName)); - logger.info(`External asset ${fileName} saved to ${PATH + fileName}`); - }); - } else { - logger.info(`Downloading external asset ${fileName} over clearnet...`); - await axios.get(url, { - responseType: 'stream', - timeout: 30000 - }).then(function (response) { - response.data.pipe(fs.createWriteStream(PATH + fileName)); - logger.info(`External asset ${fileName} saved to ${PATH + fileName}`); - }); + } catch (e: any) { + reject(e); } - } catch (e: any) { - throw new Error(`Failed to download external asset. ` + e); - } + }); } } From c51216f97c61cb2e8193c96c61e0d321a266c65b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 23 Apr 2022 08:56:55 +0900 Subject: [PATCH 2/2] Remove hardcoded liquid.network in asset icons url --- frontend/src/app/components/asset/asset.component.html | 2 +- .../components/assets/asset-group/asset-group.component.html | 2 +- .../assets/assets-featured/assets-featured.component.html | 4 ++-- frontend/src/app/dashboard/dashboard.component.html | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/asset/asset.component.html b/frontend/src/app/components/asset/asset.component.html index b1728a0ff..800e0a8f6 100644 --- a/frontend/src/app/components/asset/asset.component.html +++ b/frontend/src/app/components/asset/asset.component.html @@ -64,7 +64,7 @@
- + diff --git a/frontend/src/app/components/assets/asset-group/asset-group.component.html b/frontend/src/app/components/assets/asset-group/asset-group.component.html index ac0ed4327..df3f90abd 100644 --- a/frontend/src/app/components/assets/asset-group/asset-group.component.html +++ b/frontend/src/app/components/assets/asset-group/asset-group.component.html @@ -15,7 +15,7 @@
- +
{{ asset.name }} diff --git a/frontend/src/app/components/assets/assets-featured/assets-featured.component.html b/frontend/src/app/components/assets/assets-featured/assets-featured.component.html index b87713ceb..7893fda2c 100644 --- a/frontend/src/app/components/assets/assets-featured/assets-featured.component.html +++ b/frontend/src/app/components/assets/assets-featured/assets-featured.component.html @@ -3,14 +3,14 @@
- +
{{ group.name }}
Group of {{ group.assets.length | number }} assets
 - +
{{ group.name }} diff --git a/frontend/src/app/dashboard/dashboard.component.html b/frontend/src/app/dashboard/dashboard.component.html index 95ff4aa33..80ab9545d 100644 --- a/frontend/src/app/dashboard/dashboard.component.html +++ b/frontend/src/app/dashboard/dashboard.component.html @@ -79,7 +79,7 @@ - +