From 22665f149b9e172ea0bc85a3eeacc44be914b3e5 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 29 Jul 2023 14:23:06 +0900 Subject: [PATCH 01/15] add /prices api endpoint --- backend/src/api/prices/prices.routes.ts | 19 +++++++++++++++++++ backend/src/index.ts | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 backend/src/api/prices/prices.routes.ts diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts new file mode 100644 index 000000000..03951ec05 --- /dev/null +++ b/backend/src/api/prices/prices.routes.ts @@ -0,0 +1,19 @@ +import { Application, Request, Response } from 'express'; +import config from '../../config'; +import pricesUpdater from '../../tasks/price-updater'; + +class PricesRoutes { + public initRoutes(app: Application): void { + app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this)); + } + + private async $getCurrentPrices(req: Request, res: Response): Promise { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + + res.json(pricesUpdater.getLatestPrices()); + } +} + +export default new PricesRoutes(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 185a47067..b00909bc6 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -30,6 +30,7 @@ import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; import networkSyncService from './tasks/lightning/network-sync.service'; import statisticsRoutes from './api/statistics/statistics.routes'; +import pricesRoutes from './api/prices/prices.routes'; import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; import liquidRoutes from './api/liquid/liquid.routes'; @@ -261,6 +262,7 @@ class Server { setUpHttpApiRoutes(): void { bitcoinRoutes.initRoutes(this.app); + pricesRoutes.initRoutes(this.app); if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) { statisticsRoutes.initRoutes(this.app); } From 09c23b1241332b06c80a87c35a03ff3224c490e9 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 29 Jul 2023 14:45:00 +0900 Subject: [PATCH 02/15] configurable price update frequency --- backend/mempool-config.sample.json | 1 + backend/src/__fixtures__/mempool-config.template.json | 1 + backend/src/__tests__/config.test.ts | 1 + backend/src/config.ts | 2 ++ backend/src/tasks/price-updater.ts | 4 ++-- docker/backend/mempool-config.json | 1 + docker/backend/start.sh | 2 ++ production/mempool-config.mainnet.json | 3 +++ 8 files changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7948049fc..fddd44781 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -116,6 +116,7 @@ "PASSWORD": "" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": 300, "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" }, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index ab700c466..3a2b151a5 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -93,6 +93,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index edfcc7f47..cb564a334 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -102,6 +102,7 @@ describe('Mempool Backend Config', () => { }); expect(config.PRICE_DATA_SERVER).toStrictEqual({ + UPDATE_FREQUENCY: 3600, TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 3a028d0cd..5117983fb 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -116,6 +116,7 @@ interface IConfig { PASSWORD: string; }; PRICE_DATA_SERVER: { + UPDATE_FREQUENCY: number; TOR_URL: string; CLEARNET_URL: string; }; @@ -259,6 +260,7 @@ const defaults: IConfig = { 'PASSWORD': '' }, 'PRICE_DATA_SERVER': { + 'UPDATE_FREQUENCY': 3600, 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' }, diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index fafe2b913..335a61065 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) { - // Refresh only once every hour + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) { + // Refresh every UPDATE_FREQUENCY seconds at most return; } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b47d53b8..8b8a79969 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -112,6 +112,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index e05c73710..add483568 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -95,6 +95,7 @@ __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} # PRICE_DATA_SERVER +__PRICE_DATA_SERVER_UPDATE_FREQUENCY__=${PRICE_DATA_SERVER_UPDATE_FREQUENCY:=300} __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} @@ -226,6 +227,7 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json +sed -i "s!__PRICE_DATA_SERVER_UPDATE_FREQUENCY__!${__PRICE_DATA_SERVER_UPDATE_FREQUENCY__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 5e25bcb76..769dd8019 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -49,6 +49,9 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, + "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": 300 + }, "REPLICATION": { "ENABLED": true, "AUDIT": true, From 864d9239ceacea56139b5de78e2e3c9f39660b44 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 16:29:36 +0900 Subject: [PATCH 03/15] Update backend/src/__fixtures__/mempool-config.template.json --- backend/src/__fixtures__/mempool-config.template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 3a2b151a5..5ed888962 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -93,7 +93,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY", + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, From b2d4000b2d5b7326dbab9a337b0a469e7b4f478f Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 16:34:47 +0900 Subject: [PATCH 04/15] Update backend/src/api/prices/prices.routes.ts --- backend/src/api/prices/prices.routes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts index 03951ec05..c6a643dd8 100644 --- a/backend/src/api/prices/prices.routes.ts +++ b/backend/src/api/prices/prices.routes.ts @@ -7,7 +7,7 @@ class PricesRoutes { app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this)); } - private async $getCurrentPrices(req: Request, res: Response): Promise { + private $getCurrentPrices(req: Request, res: Response): void { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); From ae59f95ba9bdbfde43714c173a5af5ae3b846715 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 19:27:19 +0900 Subject: [PATCH 05/15] Refactoring price update config. Fixing last price time. --- backend/mempool-config.sample.json | 8 ++------ .../src/__fixtures__/mempool-config.template.json | 12 ++++-------- backend/src/__tests__/config.test.ts | 9 +-------- backend/src/config.ts | 14 ++------------ backend/src/indexer.ts | 8 ++++++-- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 10 +++++----- docker/backend/mempool-config.json | 8 ++------ production/mempool-config.mainnet.json | 6 ++---- 9 files changed, 25 insertions(+), 52 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index fddd44781..7f8bb4e22 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -32,7 +32,8 @@ "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", @@ -115,11 +116,6 @@ "USERNAME": "", "PASSWORD": "" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300, - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "https://mempool.space/api/v1", "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 5ed888962..583f8fb2d 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -23,8 +23,8 @@ "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": 14, - "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__POOLS_JSON_URL__", + "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "AUDIT": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": true, @@ -33,7 +33,8 @@ "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -92,11 +93,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index cb564a334..c42ccd223 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, + PRICE_UPDATE_FREQUENCY: 3600, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); @@ -101,12 +102,6 @@ describe('Mempool Backend Config', () => { PASSWORD: '' }); - expect(config.PRICE_DATA_SERVER).toStrictEqual({ - UPDATE_FREQUENCY: 3600, - TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' - }); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({ MEMPOOL_API: 'https://mempool.space/api/v1', MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -164,8 +159,6 @@ describe('Mempool Backend Config', () => { expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY); - expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); expect(config.REDIS).toStrictEqual(fixture.REDIS); diff --git a/backend/src/config.ts b/backend/src/config.ts index 5117983fb..1c59bcc46 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; + PRICE_UPDATE_FREQUENCY: number; }; ESPLORA: { REST_API_URL: string; @@ -115,11 +116,6 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; - PRICE_DATA_SERVER: { - UPDATE_FREQUENCY: number; - TOR_URL: string; - CLEARNET_URL: string; - }; EXTERNAL_DATA_SERVER: { MEMPOOL_API: string; MEMPOOL_ONION: string; @@ -182,6 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, + 'PRICE_UPDATE_FREQUENCY': 3600, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', @@ -259,11 +256,6 @@ const defaults: IConfig = { 'USERNAME': '', 'PASSWORD': '' }, - 'PRICE_DATA_SERVER': { - 'UPDATE_FREQUENCY': 3600, - 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' - }, 'EXTERNAL_DATA_SERVER': { 'MEMPOOL_API': 'https://mempool.space/api/v1', 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -304,7 +296,6 @@ class Config implements IConfig { LND: IConfig['LND']; CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; - PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; @@ -325,7 +316,6 @@ class Config implements IConfig { this.LND = configs.LND; this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; - this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index d89a2647f..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -105,6 +105,12 @@ class Indexer { return; } + try { + await priceUpdater.$run(); + } catch (e) { + logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); + } + // Do not attempt to index anything unless Bitcoin Core is fully synced const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { @@ -119,8 +125,6 @@ class Indexer { await this.checkAvailableCoreIndexes(); try { - await priceUpdater.$run(); - const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index ed9d1fd72..7d77834f5 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 335a61065..91a36d04b 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) { - // Refresh every UPDATE_FREQUENCY seconds at most + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) { + // Refresh every PRICE_UPDATE_FREQUENCY seconds at most return; } @@ -146,14 +146,11 @@ class PriceUpdater { } } - logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); - if (config.DATABASE.ENABLED === true) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 - this.latestPrices.time = nowRounded.getTime() / 1000; await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { this.lastRun = previousRun + 5 * 60; @@ -161,6 +158,9 @@ class PriceUpdater { } } + this.latestPrices.time = Math.round(new Date().getTime() / 1000); + logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); + if (this.ratesChangedCallback) { this.ratesChangedCallback(this.latestPrices); } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b8a79969..19d7965fd 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -33,7 +33,8 @@ "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__, "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__" + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", + "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -111,11 +112,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 769dd8019..02078cf78 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -18,7 +18,8 @@ "USE_SECOND_NODE_FOR_MINFEE": true, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 300 }, "SYSLOG" : { "MIN_PRIORITY": "debug" @@ -49,9 +50,6 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300 - }, "REPLICATION": { "ENABLED": true, "AUDIT": true, From bcb3b39bd87e6c621db7a9a3f7e51e73689ddaed Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 12:09:05 +0900 Subject: [PATCH 06/15] Updating start.sh and README --- docker/README.md | 4 +++- docker/backend/start.sh | 12 ++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docker/README.md b/docker/README.md index d95bc7aee..f70b600e1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -113,7 +113,8 @@ Below we list all settings from `mempool-config.json` and the corresponding over "ADVANCED_GBT_MEMPOOL": false, "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, - "DISK_CACHE_BLOCK_INTERVAL": 6 + "DISK_CACHE_BLOCK_INTERVAL": 6, + "PRICE_UPDATE_FREQUENCY": 3600 }, ``` @@ -146,6 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" + MEMPOOL_PRICE_UPDATE_FREQUENCY: "" ... ``` diff --git a/docker/backend/start.sh b/docker/backend/start.sh index add483568..8f8c79b41 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} - +__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -94,11 +94,6 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} -# PRICE_DATA_SERVER -__PRICE_DATA_SERVER_UPDATE_FREQUENCY__=${PRICE_DATA_SERVER_UPDATE_FREQUENCY:=300} -__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} -__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} - # EXTERNAL_DATA_SERVER __EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} __EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} @@ -178,6 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__} sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json @@ -227,10 +223,6 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_UPDATE_FREQUENCY__!${__PRICE_DATA_SERVER_UPDATE_FREQUENCY__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json - sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json From 83a487ecae0193e871db3b491301e15037caac5f Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 13:52:47 +0900 Subject: [PATCH 07/15] Setting fixture to fixed number --- backend/src/__fixtures__/mempool-config.template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 583f8fb2d..e3e8f5b14 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -34,7 +34,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__" + "PRICE_UPDATE_FREQUENCY": 3600 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", From af4d0b4d3f6329c41c3f0742482440a6630bbed0 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 15:30:23 +0900 Subject: [PATCH 08/15] Allow priceUpdater to run without storing to database --- backend/src/index.ts | 1 + backend/src/indexer.ts | 2 +- backend/src/tasks/price-updater.ts | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index b00909bc6..adb3f2e02 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -194,6 +194,7 @@ class Server { await memPool.$updateMempool(newMempool, pollRate); } indexer.$run(); + priceUpdater.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 7ec65d9c9..392174ecf 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -106,7 +106,7 @@ class Indexer { } try { - await priceUpdater.$run(); + await priceUpdater.$run(true); } catch (e) { logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 91a36d04b..f96616c12 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -72,7 +72,7 @@ class PriceUpdater { this.latestPrices = await PricesRepository.$getLatestConversionRates(); } - public async $run(): Promise { + public async $run(storeInDb: boolean = false): Promise { if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') { // Coins have no value on testnet/signet, so we want to always show 0 return; @@ -89,7 +89,7 @@ class PriceUpdater { } try { - await this.$updatePrice(); + await this.$updatePrice(storeInDb); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } @@ -103,7 +103,7 @@ class PriceUpdater { /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ - private async $updatePrice(): Promise { + private async $updatePrice(storeInDb: boolean): Promise { if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { this.lastRun = await PricesRepository.$getLatestPriceTime(); } @@ -146,7 +146,7 @@ class PriceUpdater { } } - if (config.DATABASE.ENABLED === true) { + if (config.DATABASE.ENABLED === true && storeInDb) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour From 284d39baa8046bf11280562e75b0c4bf9927bfb4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 16:35:37 +0900 Subject: [PATCH 09/15] Update price at least every hour --- backend/src/tasks/price-updater.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index f96616c12..05d6f0bec 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) { - // Refresh every PRICE_UPDATE_FREQUENCY seconds at most + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) { + // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds return; } From b213f43a91c2dde554479935c6a299807a508452 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 16:37:24 +0900 Subject: [PATCH 10/15] Erasing unused PRICE_DATA_SERVER from readme --- docker/README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/docker/README.md b/docker/README.md index f70b600e1..4270d11c8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -365,25 +365,6 @@ Corresponding `docker-compose.yml` overrides:
-`mempool-config.json`: -```json - "PRICE_DATA_SERVER": { - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - } -``` - -Corresponding `docker-compose.yml` overrides: -```yaml - api: - environment: - PRICE_DATA_SERVER_TOR_URL: "" - PRICE_DATA_SERVER_CLEARNET_URL: "" - ... -``` - -
- `mempool-config.json`: ```json "LIGHTNING": { From 172c77328be75d13389f5cb2251269f702355a60 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 3 Aug 2023 17:42:41 +0900 Subject: [PATCH 11/15] Changing setting to per hour --- backend/mempool-config.sample.json | 2 +- .../__fixtures__/mempool-config.template.json | 2 +- backend/src/__tests__/config.test.ts | 2 +- backend/src/config.ts | 4 +- backend/src/indexer.ts | 2 +- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 56 ++++++++++++++----- docker/README.md | 4 +- docker/backend/mempool-config.json | 2 +- docker/backend/start.sh | 4 +- production/mempool-config.mainnet.json | 2 +- 11 files changed, 55 insertions(+), 27 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7f8bb4e22..8fd59a430 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -33,7 +33,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index e3e8f5b14..471b7bb47 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -34,7 +34,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index c42ccd223..9cd547b90 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, - PRICE_UPDATE_FREQUENCY: 3600, + PRICE_UPDATES_PER_HOUR: 1, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 1c59bcc46..2e89bd525 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,7 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; - PRICE_UPDATE_FREQUENCY: number; + PRICE_UPDATES_PER_HOUR: number; }; ESPLORA: { REST_API_URL: string; @@ -178,7 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, - 'PRICE_UPDATE_FREQUENCY': 3600, + 'PRICE_UPDATES_PER_HOUR': 1, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 392174ecf..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -106,7 +106,7 @@ class Indexer { } try { - await priceUpdater.$run(true); + await priceUpdater.$run(); } catch (e) { logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 7d77834f5..ed9d1fd72 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 05d6f0bec..bbfb7d167 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,9 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private lastRun = 0; + private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private cyclePosition = -1; + private firstRun = true; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -41,6 +43,8 @@ class PriceUpdater { this.feeds.push(new CoinbaseApi()); this.feeds.push(new BitfinexApi()); this.feeds.push(new GeminiApi()); + + this.setCyclePosition(); } public getLatestPrices(): ApiPrice { @@ -72,7 +76,7 @@ class PriceUpdater { this.latestPrices = await PricesRepository.$getLatestConversionRates(); } - public async $run(storeInDb: boolean = false): Promise { + public async $run(): Promise { if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') { // Coins have no value on testnet/signet, so we want to always show 0 return; @@ -89,7 +93,7 @@ class PriceUpdater { } try { - await this.$updatePrice(storeInDb); + await this.$updatePrice(); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } @@ -100,22 +104,42 @@ class PriceUpdater { this.running = false; } + private getMillisecondsSinceBeginningOfHour(): number { + const now = new Date(); + const beginningOfHour = new Date(now); + beginningOfHour.setMinutes(0, 0, 0); + return now.getTime() - beginningOfHour.getTime(); + } + + private setCyclePosition(): void { + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) { + if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) { + this.cyclePosition = i; + return; + } + } + this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + } + /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ - private async $updatePrice(storeInDb: boolean): Promise { - if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { - this.lastRun = await PricesRepository.$getLatestPriceTime(); + private async $updatePrice(): Promise { + let forceUpdate = false; + if (this.firstRun === true && config.DATABASE.ENABLED === true) { + const lastUpdate = await PricesRepository.$getLatestPriceTime(); + if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) { + forceUpdate = true; + } + this.firstRun = false; } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) { - // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) { return; } - const previousRun = this.lastRun; - this.lastRun = new Date().getTime() / 1000; - for (const currency of this.currencies) { let prices: number[] = []; @@ -146,14 +170,13 @@ class PriceUpdater { } } - if (config.DATABASE.ENABLED === true && storeInDb) { + if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { - this.lastRun = previousRun + 5 * 60; logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`); } } @@ -165,7 +188,12 @@ class PriceUpdater { this.ratesChangedCallback(this.latestPrices); } - this.lastRun = new Date().getTime() / 1000; + if (!forceUpdate) { + this.cyclePosition++; + if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) { + this.cyclePosition = 0; + } + } if (this.latestPrices.USD === -1) { this.latestPrices = await PricesRepository.$getLatestConversionRates(); diff --git a/docker/README.md b/docker/README.md index 4270d11c8..13bda7ec6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -114,7 +114,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, "DISK_CACHE_BLOCK_INTERVAL": 6, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, ``` @@ -147,7 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" - MEMPOOL_PRICE_UPDATE_FREQUENCY: "" + MEMPOOL_PRICE_UPDATES_PER_HOUR: "" ... ``` diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 19d7965fd..216ad95c2 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -34,7 +34,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", - "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 8f8c79b41..30d2ac332 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} -__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600} +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -173,7 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__} sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json -sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 02078cf78..f54635415 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -19,7 +19,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 300 + "PRICE_UPDATES_PER_HOUR": 12 }, "SYSLOG" : { "MIN_PRIORITY": "debug" From 813f3dc09deb7d4ada81c3776d253615e2fd0c55 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 3 Aug 2023 18:26:24 +0900 Subject: [PATCH 12/15] Updating config sample --- backend/mempool-config.sample.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 8fd59a430..c53139b0b 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -33,7 +33,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATES_PER_HOUR": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "127.0.0.1", From 2892bfa1d81299474ddb3de265a1ecf785b1736d Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 13:23:09 +0900 Subject: [PATCH 13/15] Fixing cycle reset at top of the hour --- backend/src/tasks/price-updater.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index bbfb7d167..79173ad37 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -28,6 +28,7 @@ class PriceUpdater { private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; + private lastTime = -1; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -136,7 +137,13 @@ class PriceUpdater { } const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); - if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) { + + // Reset the cycle on new hour + if (this.lastTime > millisecondsSinceBeginningOfHour) { + this.cyclePosition = 0; + } + this.lastTime = millisecondsSinceBeginningOfHour; + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate && this.cyclePosition !== 0) { return; } @@ -190,9 +197,6 @@ class PriceUpdater { if (!forceUpdate) { this.cyclePosition++; - if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) { - this.cyclePosition = 0; - } } if (this.latestPrices.USD === -1) { From 07f95acc29440905b2c70d992f692a3157c605cb Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 13:26:19 +0900 Subject: [PATCH 14/15] Base expiry on update frequency --- backend/src/api/prices/prices.routes.ts | 2 +- backend/src/tasks/price-updater.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts index c6a643dd8..b46331b73 100644 --- a/backend/src/api/prices/prices.routes.ts +++ b/backend/src/api/prices/prices.routes.ts @@ -10,7 +10,7 @@ class PricesRoutes { private $getCurrentPrices(req: Request, res: Response): void { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString()); res.json(pricesUpdater.getLatestPrices()); } diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 79173ad37..fd799fb87 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,7 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; private lastTime = -1; From 101de3bac7cea2596b0113be20b943a5ddc6ecdb Mon Sep 17 00:00:00 2001 From: mononaut <83316221+mononaut@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:23:55 +0900 Subject: [PATCH 15/15] Fix PRICE_UPDATES_PER_HOUR docker default Co-authored-by: nymkappa <9780671+nymkappa@users.noreply.github.com> --- docker/backend/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 30d2ac332..008c0aaf5 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} -__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600} +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}