diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 571cc0f3b..b1a2da18f 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -17,8 +17,6 @@ import { prepareBlock } from '../utils/blocks-utils'; import BlocksRepository from '../repositories/BlocksRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; -import fiatConversion from './fiat-conversion'; -import RatesRepository from '../repositories/RatesRepository'; import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; @@ -461,9 +459,6 @@ class Blocks { } } } - if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) { - await RatesRepository.$saveRate(blockExtended.height, fiatConversion.getConversionRates()); - } if (block.height % 2016 === 0) { this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 2b7b6ddea..f0f375309 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 20; + private static currentVersion = 21; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -221,6 +221,11 @@ class DatabaseMigration { if (databaseSchemaVersion < 20 && isBitcoin === true) { await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries')); } + + if (databaseSchemaVersion < 21) { + await this.$executeQuery('DROP TABLE IF EXISTS `rates`'); + await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices')); + } } catch (e) { throw e; } @@ -526,8 +531,16 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreatePricesTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS prices ( + time timestamp NOT NULL, + avg_prices JSON NOT NULL, + PRIMARY KEY (time) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { - const allowedTables = ['blocks', 'hashrates']; + const allowedTables = ['blocks', 'hashrates', 'prices']; try { for (const table of tables) { diff --git a/backend/src/index.ts b/backend/src/index.ts index 2d1438842..1b7568d2e 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -27,6 +27,7 @@ import icons from './api/liquid/icons'; import { Common } from './api/common'; import poolsUpdater from './tasks/pools-updater'; import indexer from './indexer'; +import priceUpdater from './tasks/price-updater'; class Server { private wss: WebSocket.Server | undefined; @@ -153,6 +154,7 @@ class Server { await blocks.$updateBlocks(); await memPool.$updateMempool(); indexer.$run(); + priceUpdater.$run(); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts new file mode 100644 index 000000000..d6eaf523a --- /dev/null +++ b/backend/src/repositories/PricesRepository.ts @@ -0,0 +1,32 @@ +import DB from '../database'; +import logger from '../logger'; +import { Prices } from '../tasks/price-updater'; + +class PricesRepository { + public async $savePrices(time: number, prices: Prices): Promise { + try { + await DB.query(`INSERT INTO prices(time, avg_prices) VALUE (FROM_UNIXTIME(?), ?)`, [time, JSON.stringify(prices)]); + } catch (e: any) { + logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + public async $getOldestPriceTime(): Promise { + const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices ORDER BY time LIMIT 1`); + return oldestRow[0] ? oldestRow[0].time : 0; + } + + public async $getLatestPriceTime(): Promise { + const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices ORDER BY time DESC LIMIT 1`); + return oldestRow[0] ? oldestRow[0].time : 0; + } + + public async $getPricesTimes(): Promise { + const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices`); + return times.map(time => time.time); + } +} + +export default new PricesRepository(); + diff --git a/backend/src/repositories/RatesRepository.ts b/backend/src/repositories/RatesRepository.ts deleted file mode 100644 index e84ef2827..000000000 --- a/backend/src/repositories/RatesRepository.ts +++ /dev/null @@ -1,21 +0,0 @@ -import DB from '../database'; -import logger from '../logger'; -import { IConversionRates } from '../mempool.interfaces'; - -class RatesRepository { - public async $saveRate(height: number, rates: IConversionRates) { - try { - await DB.query(`INSERT INTO rates(height, bisq_rates) VALUE (?, ?)`, [height, JSON.stringify(rates)]); - } catch (e: any) { - if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart - logger.debug(`Rate already exists for block ${height}, ignoring`); - } else { - logger.err(`Cannot save exchange rate into db for block ${height} Reason: ` + (e instanceof Error ? e.message : e)); - throw e; - } - } - } -} - -export default new RatesRepository(); - diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 1d3fec312..04d9d5d07 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -47,7 +47,7 @@ class PoolsUpdater { } if (config.DATABASE.ENABLED === true) { - this.currentSha = await this.getShaFromDb(); + this.currentSha = await this.getShaFromDb(); } logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`); @@ -70,7 +70,7 @@ class PoolsUpdater { } catch (e) { this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week - logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e)); } } @@ -84,7 +84,7 @@ class PoolsUpdater { await DB.query('DELETE FROM state where name="pools_json_sha"'); await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`); } catch (e) { - logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e)); } } } @@ -97,7 +97,7 @@ class PoolsUpdater { const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"'); return (rows.length > 0 ? rows[0].string : undefined); } catch (e) { - logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e)); return undefined; } } @@ -130,7 +130,7 @@ class PoolsUpdater { }; timeout: number; httpsAgent?: https.Agent; - } + }; const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); const axiosOptions: axiosOptions = { headers: { @@ -140,7 +140,7 @@ class PoolsUpdater { }; let retry = 0; - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { if (config.SOCKS5PROXY.ENABLED) { const socksOptions: any = { @@ -161,14 +161,14 @@ class PoolsUpdater { axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); } - + const data: AxiosResponse = await axios.get(path, axiosOptions); if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`); } return data.data; } catch (e) { - logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); retry++; } await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); diff --git a/backend/src/tasks/price-feeds/bitfinex-api.ts b/backend/src/tasks/price-feeds/bitfinex-api.ts new file mode 100644 index 000000000..be3f5617b --- /dev/null +++ b/backend/src/tasks/price-feeds/bitfinex-api.ts @@ -0,0 +1,43 @@ +import { query } from '../../utils/axios-query'; +import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater'; + +class BitfinexApi implements PriceFeed { + public name: string = 'Bitfinex'; + public currencies: string[] = ['USD', 'EUR', 'GPB', 'JPY']; + + public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC'; + public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist'; + + constructor() { + } + + public async $fetchPrice(currency): Promise { + const response = await query(this.url + currency); + return response ? parseInt(response['last_price'], 10) : -1; + } + + public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + const priceHistory: PriceHistory = {}; + + for (const currency of currencies) { + if (this.currencies.includes(currency) === false) { + continue; + } + + const response = await query(this.urlHist.replace('{GRANULARITY}', '1h').replace('{CURRENCY}', currency)); + const pricesRaw = response ? response : []; + + for (const price of pricesRaw as any[]) { + const time = Math.round(price[0] / 1000); + if (priceHistory[time] === undefined) { + priceHistory[time] = priceUpdater.getEmptyPricesObj(); + } + priceHistory[time][currency] = price[2]; + } + } + + return priceHistory; + } +} + +export default BitfinexApi; diff --git a/backend/src/tasks/price-feeds/bitflyer-api.ts b/backend/src/tasks/price-feeds/bitflyer-api.ts new file mode 100644 index 000000000..d87661abb --- /dev/null +++ b/backend/src/tasks/price-feeds/bitflyer-api.ts @@ -0,0 +1,24 @@ +import { query } from '../../utils/axios-query'; +import { PriceFeed, PriceHistory } from '../price-updater'; + +class BitflyerApi implements PriceFeed { + public name: string = 'Bitflyer'; + public currencies: string[] = ['USD', 'EUR', 'JPY']; + + public url: string = 'https://api.bitflyer.com/v1/ticker?product_code=BTC_'; + public urlHist: string = ''; + + constructor() { + } + + public async $fetchPrice(currency): Promise { + const response = await query(this.url + currency); + return response ? parseInt(response['ltp'], 10) : -1; + } + + public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + return []; + } +} + +export default BitflyerApi; diff --git a/backend/src/tasks/price-feeds/coinbase-api.ts b/backend/src/tasks/price-feeds/coinbase-api.ts new file mode 100644 index 000000000..b9abf860e --- /dev/null +++ b/backend/src/tasks/price-feeds/coinbase-api.ts @@ -0,0 +1,42 @@ +import { query } from '../../utils/axios-query'; +import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater'; + +class CoinbaseApi implements PriceFeed { + public name: string = 'Coinbase'; + public currencies: string[] = ['USD', 'EUR', 'GBP']; + + public url: string = 'https://api.coinbase.com/v2/prices/spot?currency='; + public urlHist: string = 'https://api.exchange.coinbase.com/products/BTC-{CURRENCY}/candles?granularity={GRANULARITY}'; + + constructor() { + } + + public async $fetchPrice(currency): Promise { + const response = await query(this.url + currency); + return response ? parseInt(response['data']['amount'], 10) : -1; + } + + public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + const priceHistory: PriceHistory = {}; + + for (const currency of currencies) { + if (this.currencies.includes(currency) === false) { + continue; + } + + const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency)); + const pricesRaw = response ? response : []; + + for (const price of pricesRaw as any[]) { + if (priceHistory[price[0]] === undefined) { + priceHistory[price[0]] = priceUpdater.getEmptyPricesObj(); + } + priceHistory[price[0]][currency] = price[4]; + } + } + + return priceHistory; + } +} + +export default CoinbaseApi; diff --git a/backend/src/tasks/price-feeds/ftx-api.ts b/backend/src/tasks/price-feeds/ftx-api.ts new file mode 100644 index 000000000..db58c8800 --- /dev/null +++ b/backend/src/tasks/price-feeds/ftx-api.ts @@ -0,0 +1,43 @@ +import { query } from '../../utils/axios-query'; +import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater'; + +class FtxApi implements PriceFeed { + public name: string = 'FTX'; + public currencies: string[] = ['USD', 'BRZ', 'EUR', 'JPY', 'AUD']; + + public url: string = 'https://ftx.com/api/markets/BTC/'; + public urlHist: string = 'https://ftx.com/api/markets/BTC/{CURRENCY}/candles?resolution={GRANULARITY}'; + + constructor() { + } + + public async $fetchPrice(currency): Promise { + const response = await query(this.url + currency); + return response ? parseInt(response['result']['last'], 10) : -1; + } + + public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + const priceHistory: PriceHistory = {}; + + for (const currency of currencies) { + if (this.currencies.includes(currency) === false) { + continue; + } + + const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency)); + const pricesRaw = response ? response['result'] : []; + + for (const price of pricesRaw as any[]) { + const time = Math.round(price['time'] / 1000); + if (priceHistory[time] === undefined) { + priceHistory[time] = priceUpdater.getEmptyPricesObj(); + } + priceHistory[time][currency] = price['close']; + } + } + + return priceHistory; + } +} + +export default FtxApi; diff --git a/backend/src/tasks/price-feeds/gemini-api.ts b/backend/src/tasks/price-feeds/gemini-api.ts new file mode 100644 index 000000000..6b5742a7a --- /dev/null +++ b/backend/src/tasks/price-feeds/gemini-api.ts @@ -0,0 +1,43 @@ +import { query } from '../../utils/axios-query'; +import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater'; + +class GeminiApi implements PriceFeed { + public name: string = 'Gemini'; + public currencies: string[] = ['USD', 'EUR', 'GBP', 'SGD']; + + public url: string = 'https://api.gemini.com/v1/pubticker/BTC'; + public urlHist: string = 'https://api.gemini.com/v2/candles/BTC{CURRENCY}/{GRANULARITY}'; + + constructor() { + } + + public async $fetchPrice(currency): Promise { + const response = await query(this.url + currency); + return response ? parseInt(response['last'], 10) : -1; + } + + public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + const priceHistory: PriceHistory = {}; + + for (const currency of currencies) { + if (this.currencies.includes(currency) === false) { + continue; + } + + const response = await query(this.urlHist.replace('{GRANULARITY}', '1hr').replace('{CURRENCY}', currency)); + const pricesRaw = response ? response : []; + + for (const price of pricesRaw as any[]) { + const time = Math.round(price[0] / 1000); + if (priceHistory[time] === undefined) { + priceHistory[time] = priceUpdater.getEmptyPricesObj(); + } + priceHistory[time][currency] = price[4]; + } + } + + return priceHistory; + } +} + +export default GeminiApi; diff --git a/backend/src/tasks/price-feeds/kraken-api.ts b/backend/src/tasks/price-feeds/kraken-api.ts new file mode 100644 index 000000000..02d0d3af0 --- /dev/null +++ b/backend/src/tasks/price-feeds/kraken-api.ts @@ -0,0 +1,95 @@ +import logger from '../../logger'; +import PricesRepository from '../../repositories/PricesRepository'; +import { query } from '../../utils/axios-query'; +import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater'; + +class KrakenApi implements PriceFeed { + public name: string = 'Kraken'; + public currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; + + public url: string = 'https://api.kraken.com/0/public/Ticker?pair=XBT'; + public urlHist: string = 'https://api.kraken.com/0/public/OHLC?interval={GRANULARITY}&pair=XBT'; + + constructor() { + } + + private getTicker(currency) { + let ticker = `XXBTZ${currency}`; + if (['CHF', 'AUD'].includes(currency)) { + ticker = `XBT${currency}`; + } + return ticker; + } + + public async $fetchPrice(currency): Promise { + const response = await query(this.url + currency); + return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1; + } + + public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + const priceHistory: PriceHistory = {}; + + for (const currency of currencies) { + if (this.currencies.includes(currency) === false) { + continue; + } + + const response = await query(this.urlHist.replace('{GRANULARITY}', '60') + currency); + const pricesRaw = response ? response['result'][this.getTicker(currency)] : []; + + for (const price of pricesRaw) { + if (priceHistory[price[0]] === undefined) { + priceHistory[price[0]] = priceUpdater.getEmptyPricesObj(); + } + priceHistory[price[0]][currency] = price[4]; + } + } + + return priceHistory; + } + + /** + * Fetch weekly price and save it into the database + */ + public async $insertHistoricalPrice(): Promise { + const existingPriceTimes = await PricesRepository.$getPricesTimes(); + + // EUR weekly price history goes back to timestamp 1378339200 (September 5, 2013) + // USD weekly price history goes back to timestamp 1380758400 (October 3, 2013) + // GBP weekly price history goes back to timestamp 1415232000 (November 6, 2014) + // JPY weekly price history goes back to timestamp 1415232000 (November 6, 2014) + // CAD weekly price history goes back to timestamp 1436400000 (July 9, 2015) + // CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019) + // AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020) + + const priceHistory: any = {}; // map: timestamp -> Prices + + for (const currency of this.currencies) { + const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency); + const priceHistoryRaw = response ? response['result'][this.getTicker(currency)] : []; + + for (const price of priceHistoryRaw) { + if (existingPriceTimes.includes(parseInt(price[0]))) { + continue; + } + + // prices[0] = kraken price timestamp + // prices[4] = closing price + if (priceHistory[price[0]] === undefined) { + priceHistory[price[0]] = priceUpdater.getEmptyPricesObj(); + } + priceHistory[price[0]][currency] = price[4]; + } + } + + for (const time in priceHistory) { + await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]); + } + + if (Object.keys(priceHistory).length > 0) { + logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`); + } + } +} + +export default KrakenApi; diff --git a/backend/src/tasks/price-feeds/mtgox-weekly.json b/backend/src/tasks/price-feeds/mtgox-weekly.json new file mode 100644 index 000000000..84e077ce0 --- /dev/null +++ b/backend/src/tasks/price-feeds/mtgox-weekly.json @@ -0,0 +1,762 @@ +[ + { + "ct": 1279497600, + "c": "0.08584" + }, + { + "ct": 1280102400, + "c": "0.0505" + }, + { + "ct": 1280707200, + "c": "0.0611" + }, + { + "ct": 1281312000, + "c": "0.0609" + }, + { + "ct": 1281916800, + "c": "0.06529" + }, + { + "ct": 1282521600, + "c": "0.066" + }, + { + "ct": 1283126400, + "c": "0.064" + }, + { + "ct": 1283731200, + "c": "0.06165" + }, + { + "ct": 1284336000, + "c": "0.0615" + }, + { + "ct": 1284940800, + "c": "0.0627" + }, + { + "ct": 1285545600, + "c": "0.0622" + }, + { + "ct": 1286150400, + "c": "0.06111" + }, + { + "ct": 1286755200, + "c": "0.0965" + }, + { + "ct": 1287360000, + "c": "0.102" + }, + { + "ct": 1287964800, + "c": "0.11501" + }, + { + "ct": 1288569600, + "c": "0.1925" + }, + { + "ct": 1289174400, + "c": "0.34" + }, + { + "ct": 1289779200, + "c": "0.27904" + }, + { + "ct": 1290384000, + "c": "0.27675" + }, + { + "ct": 1290988800, + "c": "0.27" + }, + { + "ct": 1291593600, + "c": "0.19" + }, + { + "ct": 1292198400, + "c": "0.2189" + }, + { + "ct": 1292803200, + "c": "0.2401" + }, + { + "ct": 1293408000, + "c": "0.263" + }, + { + "ct": 1294012800, + "c": "0.29997" + }, + { + "ct": 1294617600, + "c": "0.323" + }, + { + "ct": 1295222400, + "c": "0.38679" + }, + { + "ct": 1295827200, + "c": "0.4424" + }, + { + "ct": 1296432000, + "c": "0.4799" + }, + { + "ct": 1297036800, + "c": "0.8968" + }, + { + "ct": 1297641600, + "c": "1.05" + }, + { + "ct": 1298246400, + "c": "0.865" + }, + { + "ct": 1298851200, + "c": "0.89" + }, + { + "ct": 1299456000, + "c": "0.8999" + }, + { + "ct": 1300060800, + "c": "0.89249" + }, + { + "ct": 1300665600, + "c": "0.75218" + }, + { + "ct": 1301270400, + "c": "0.82754" + }, + { + "ct": 1301875200, + "c": "0.779" + }, + { + "ct": 1302480000, + "c": "0.7369" + }, + { + "ct": 1303084800, + "c": "1.1123" + }, + { + "ct": 1303689600, + "c": "1.6311" + }, + { + "ct": 1304294400, + "c": "3.03311" + }, + { + "ct": 1304899200, + "c": "3.8659" + }, + { + "ct": 1305504000, + "c": "6.98701" + }, + { + "ct": 1306108800, + "c": "6.6901" + }, + { + "ct": 1306713600, + "c": "8.4" + }, + { + "ct": 1307318400, + "c": "16.7" + }, + { + "ct": 1307923200, + "c": "18.5464" + }, + { + "ct": 1308528000, + "c": "17.51" + }, + { + "ct": 1309132800, + "c": "16.45001" + }, + { + "ct": 1309737600, + "c": "15.44049" + }, + { + "ct": 1310342400, + "c": "14.879" + }, + { + "ct": 1310947200, + "c": "13.16" + }, + { + "ct": 1311552000, + "c": "13.98001" + }, + { + "ct": 1312156800, + "c": "13.35" + }, + { + "ct": 1312761600, + "c": "7.9" + }, + { + "ct": 1313366400, + "c": "10.7957" + }, + { + "ct": 1313971200, + "c": "11.31125" + }, + { + "ct": 1314576000, + "c": "9.07011" + }, + { + "ct": 1315180800, + "c": "8.17798" + }, + { + "ct": 1315785600, + "c": "5.86436" + }, + { + "ct": 1316390400, + "c": "5.2" + }, + { + "ct": 1316995200, + "c": "5.33" + }, + { + "ct": 1317600000, + "c": "5.02701" + }, + { + "ct": 1318204800, + "c": "4.10288" + }, + { + "ct": 1318809600, + "c": "3.5574" + }, + { + "ct": 1319414400, + "c": "3.12657" + }, + { + "ct": 1320019200, + "c": "3.27" + }, + { + "ct": 1320624000, + "c": "2.95959" + }, + { + "ct": 1321228800, + "c": "2.99626" + }, + { + "ct": 1321833600, + "c": "2.2" + }, + { + "ct": 1322438400, + "c": "2.47991" + }, + { + "ct": 1323043200, + "c": "2.82809" + }, + { + "ct": 1323648000, + "c": "3.2511" + }, + { + "ct": 1324252800, + "c": "3.193" + }, + { + "ct": 1324857600, + "c": "4.225" + }, + { + "ct": 1325462400, + "c": "5.26766" + }, + { + "ct": 1326067200, + "c": "7.11358" + }, + { + "ct": 1326672000, + "c": "7.00177" + }, + { + "ct": 1327276800, + "c": "6.3097" + }, + { + "ct": 1327881600, + "c": "5.38191" + }, + { + "ct": 1328486400, + "c": "5.68881" + }, + { + "ct": 1329091200, + "c": "5.51468" + }, + { + "ct": 1329696000, + "c": "4.38669" + }, + { + "ct": 1330300800, + "c": "4.922" + }, + { + "ct": 1330905600, + "c": "4.8201" + }, + { + "ct": 1331510400, + "c": "4.90901" + }, + { + "ct": 1332115200, + "c": "5.27943" + }, + { + "ct": 1332720000, + "c": "4.55001" + }, + { + "ct": 1333324800, + "c": "4.81922" + }, + { + "ct": 1333929600, + "c": "4.79253" + }, + { + "ct": 1334534400, + "c": "4.96892" + }, + { + "ct": 1335139200, + "c": "5.20352" + }, + { + "ct": 1335744000, + "c": "4.90441" + }, + { + "ct": 1336348800, + "c": "5.04991" + }, + { + "ct": 1336953600, + "c": "4.92996" + }, + { + "ct": 1337558400, + "c": "5.09002" + }, + { + "ct": 1338163200, + "c": "5.13896" + }, + { + "ct": 1338768000, + "c": "5.2051" + }, + { + "ct": 1339372800, + "c": "5.46829" + }, + { + "ct": 1339977600, + "c": "6.16382" + }, + { + "ct": 1340582400, + "c": "6.35002" + }, + { + "ct": 1341187200, + "c": "6.62898" + }, + { + "ct": 1341792000, + "c": "6.79898" + }, + { + "ct": 1342396800, + "c": "7.62101" + }, + { + "ct": 1343001600, + "c": "8.4096" + }, + { + "ct": 1343606400, + "c": "8.71027" + }, + { + "ct": 1344211200, + "c": "10.86998" + }, + { + "ct": 1344816000, + "c": "11.6239" + }, + { + "ct": 1345420800, + "c": "7.98" + }, + { + "ct": 1346025600, + "c": "10.61" + }, + { + "ct": 1346630400, + "c": "10.2041" + }, + { + "ct": 1347235200, + "c": "11.02" + }, + { + "ct": 1347840000, + "c": "11.87" + }, + { + "ct": 1348444800, + "c": "12.19331" + }, + { + "ct": 1349049600, + "c": "12.4" + }, + { + "ct": 1349654400, + "c": "11.8034" + }, + { + "ct": 1350259200, + "c": "11.7389" + }, + { + "ct": 1350864000, + "c": "11.63107" + }, + { + "ct": 1351468800, + "c": "10.69998" + }, + { + "ct": 1352073600, + "c": "10.80011" + }, + { + "ct": 1352678400, + "c": "10.84692" + }, + { + "ct": 1353283200, + "c": "11.65961" + }, + { + "ct": 1353888000, + "c": "12.4821" + }, + { + "ct": 1354492800, + "c": "12.50003" + }, + { + "ct": 1355097600, + "c": "13.388" + }, + { + "ct": 1355702400, + "c": "13.30002" + }, + { + "ct": 1356307200, + "c": "13.31202" + }, + { + "ct": 1356912000, + "c": "13.45001" + }, + { + "ct": 1357516800, + "c": "13.5199" + }, + { + "ct": 1358121600, + "c": "14.11601" + }, + { + "ct": 1358726400, + "c": "15.7" + }, + { + "ct": 1359331200, + "c": "17.95" + }, + { + "ct": 1359936000, + "c": "20.59" + }, + { + "ct": 1360540800, + "c": "23.96975" + }, + { + "ct": 1361145600, + "c": "26.8146" + }, + { + "ct": 1361750400, + "c": "29.88999" + }, + { + "ct": 1362355200, + "c": "34.49999" + }, + { + "ct": 1362960000, + "c": "46" + }, + { + "ct": 1363564800, + "c": "47.4" + }, + { + "ct": 1364169600, + "c": "71.93" + }, + { + "ct": 1364774400, + "c": "93.03001" + }, + { + "ct": 1365379200, + "c": "162.30102" + }, + { + "ct": 1365984000, + "c": "89.99999" + }, + { + "ct": 1366588800, + "c": "119.2" + }, + { + "ct": 1367193600, + "c": "134.44444" + }, + { + "ct": 1367798400, + "c": "115.98" + }, + { + "ct": 1368403200, + "c": "114.82002" + }, + { + "ct": 1369008000, + "c": "122.49999" + }, + { + "ct": 1369612800, + "c": "133.5" + }, + { + "ct": 1370217600, + "c": "122.5" + }, + { + "ct": 1370822400, + "c": "100.43743" + }, + { + "ct": 1371427200, + "c": "99.9" + }, + { + "ct": 1372032000, + "c": "107.90001" + }, + { + "ct": 1372636800, + "c": "97.51" + }, + { + "ct": 1373241600, + "c": "76.5" + }, + { + "ct": 1373846400, + "c": "94.41986" + }, + { + "ct": 1374451200, + "c": "91.998" + }, + { + "ct": 1375056000, + "c": "98.78008" + }, + { + "ct": 1375660800, + "c": "105.12" + }, + { + "ct": 1376265600, + "c": "105" + }, + { + "ct": 1376870400, + "c": "113.38" + }, + { + "ct": 1377475200, + "c": "122.11102" + }, + { + "ct": 1378080000, + "c": "146.01003" + }, + { + "ct": 1378684800, + "c": "126.31501" + }, + { + "ct": 1379289600, + "c": "138.3002" + }, + { + "ct": 1379894400, + "c": "134.00001" + }, + { + "ct": 1380499200, + "c": "143.88402" + }, + { + "ct": 1381104000, + "c": "137.8" + }, + { + "ct": 1381708800, + "c": "147.53" + }, + { + "ct": 1382313600, + "c": "186.1" + }, + { + "ct": 1382918400, + "c": "207.0001" + }, + { + "ct": 1383523200, + "c": "224.01001" + }, + { + "ct": 1384128000, + "c": "336.33101" + }, + { + "ct": 1384732800, + "c": "528" + }, + { + "ct": 1385337600, + "c": "795" + }, + { + "ct": 1385942400, + "c": "1004.42392" + }, + { + "ct": 1386547200, + "c": "804.5" + }, + { + "ct": 1387152000, + "c": "919.985" + }, + { + "ct": 1387756800, + "c": "639.48" + }, + { + "ct": 1388361600, + "c": "786.98" + }, + { + "ct": 1388966400, + "c": "1015" + }, + { + "ct": 1389571200, + "c": "940" + }, + { + "ct": 1390176000, + "c": "954.995" + }, + { + "ct": 1390780800, + "c": "1007.98999" + }, + { + "ct": 1391385600, + "c": "954" + }, + { + "ct": 1391990400, + "c": "659.49776" + }, + { + "ct": 1392595200, + "c": "299.702" + }, + { + "ct": 1393200000, + "c": "310.00001" + }, + { + "ct": 1393804800, + "c": "135" + } +] diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts new file mode 100644 index 000000000..254e8ef1c --- /dev/null +++ b/backend/src/tasks/price-updater.ts @@ -0,0 +1,251 @@ +import * as fs from 'fs'; +import config from '../config'; +import logger from '../logger'; +import PricesRepository from '../repositories/PricesRepository'; +import BitfinexApi from './price-feeds/bitfinex-api'; +import BitflyerApi from './price-feeds/bitflyer-api'; +import CoinbaseApi from './price-feeds/coinbase-api'; +import FtxApi from './price-feeds/ftx-api'; +import GeminiApi from './price-feeds/gemini-api'; +import KrakenApi from './price-feeds/kraken-api'; + +export interface PriceFeed { + name: string; + url: string; + urlHist: string; + currencies: string[]; + + $fetchPrice(currency): Promise; + $fetchRecentHourlyPrice(currencies: string[]): Promise; +} + +export interface PriceHistory { + [timestamp: number]: Prices; +} + +export interface Prices { + USD: number; + EUR: number; + GBP: number; + CAD: number; + CHF: number; + AUD: number; + JPY: number; +} + +class PriceUpdater { + historyInserted: boolean = false; + lastRun: number = 0; + lastHistoricalRun: number = 0; + running: boolean = false; + feeds: PriceFeed[] = []; + currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; + latestPrices: Prices; + + constructor() { + this.latestPrices = this.getEmptyPricesObj(); + + this.feeds.push(new BitflyerApi()); // Does not have historical endpoint + this.feeds.push(new FtxApi()); + this.feeds.push(new KrakenApi()); + this.feeds.push(new CoinbaseApi()); + this.feeds.push(new BitfinexApi()); + this.feeds.push(new GeminiApi()); + } + + public getEmptyPricesObj(): Prices { + return { + USD: -1, + EUR: -1, + GBP: -1, + CAD: -1, + CHF: -1, + AUD: -1, + JPY: -1, + }; + } + + public async $run(): Promise { + if (this.running === true) { + return; + } + this.running = true; + + if ((Math.round(new Date().getTime() / 1000) - this.lastHistoricalRun) > 3600 * 24) { + // Once a day, look for missing prices (could happen due to network connectivity issues) + this.historyInserted = false; + } + + try { + if (this.historyInserted === false && config.DATABASE.ENABLED === true) { + await this.$insertHistoricalPrices(); + } else { + await this.$updatePrice(); + } + } catch (e) { + logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`); + } + + this.running = false; + } + + /** + * Fetch last BTC price from exchanges, average them, and save it in the database once every hour + */ + private async $updatePrice(): Promise { + if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { + this.lastRun = await PricesRepository.$getLatestPriceTime(); + } + + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) { + // Refresh only once every hour + return; + } + + const previousRun = this.lastRun; + this.lastRun = new Date().getTime() / 1000; + + for (const currency of this.currencies) { + let prices: number[] = []; + + for (const feed of this.feeds) { + // Fetch prices from API which supports `currency` + if (feed.currencies.includes(currency)) { + try { + const price = await feed.$fetchPrice(currency); + if (price > 0) { + prices.push(price); + } + logger.debug(`${feed.name} BTC/${currency} price: ${price}`); + } catch (e) { + logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`); + } + } + } + if (prices.length === 1) { + logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`); + } + + // Compute average price, non weighted + prices = prices.filter(price => price > 0); + this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length); + } + + logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); + + if (config.DATABASE.ENABLED === true) { + // Save everything in db + try { + const p = 60 * 60 * 1000; // milliseconds in an hour + const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 + await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); + } catch (e) { + this.lastRun = previousRun + 5 * 60; + logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`); + } + } + + this.lastRun = new Date().getTime() / 1000; + } + + /** + * Called once by the database migration to initialize historical prices data (weekly) + * We use MtGox weekly price from July 19, 2010 to September 30, 2013 + * We use Kraken weekly price from October 3, 2013 up to last month + * We use Kraken hourly price for the past month + */ + private async $insertHistoricalPrices(): Promise { + const existingPriceTimes = await PricesRepository.$getPricesTimes(); + + // Insert MtGox weekly prices + const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString()); + const prices = this.getEmptyPricesObj(); + let insertedCount: number = 0; + for (const price of pricesJson) { + if (existingPriceTimes.includes(price['ct'])) { + continue; + } + + // From 1380758400 we will use Kraken price as it follows closely MtGox, but was not affected as much + // by the MtGox exchange collapse a few months later + if (price['ct'] > 1380758400) { + break; + } + prices.USD = price['c']; + await PricesRepository.$savePrices(price['ct'], prices); + ++insertedCount; + } + if (insertedCount > 0) { + logger.info(`Inserted ${insertedCount} MtGox USD weekly price history into db`); + } + + // Insert Kraken weekly prices + await new KrakenApi().$insertHistoricalPrice(); + + // Insert missing recent hourly prices + await this.$insertMissingRecentPrices(); + + this.historyInserted = true; + this.lastHistoricalRun = new Date().getTime(); + } + + /** + * Find missing hourly prices and insert them in the database + * It has a limited backward range and it depends on which API are available + */ + private async $insertMissingRecentPrices(): Promise { + const existingPriceTimes = await PricesRepository.$getPricesTimes(); + + logger.info(`Fetching hourly price history from exchanges and saving missing ones into the database, this may take a while`); + + const historicalPrices: PriceHistory[] = []; + + // Fetch all historical hourly prices + for (const feed of this.feeds) { + try { + historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies)); + } catch (e) { + logger.info(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`); + } + } + + // Group them by timestamp and currency, for example + // grouped[123456789]['USD'] = [1, 2, 3, 4]; + let grouped: Object = {}; + for (const historicalEntry of historicalPrices) { + for (const time in historicalEntry) { + if (existingPriceTimes.includes(parseInt(time, 10))) { + continue; + } + + if (grouped[time] == undefined) { + grouped[time] = { + USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: [] + } + } + + for (const currency of this.currencies) { + const price = historicalEntry[time][currency]; + if (price > 0) { + grouped[time][currency].push(parseInt(price, 10)); + } + } + } + } + + // Average prices and insert everything into the db + let totalInserted = 0; + for (const time in grouped) { + const prices: Prices = this.getEmptyPricesObj(); + for (const currency in grouped[time]) { + prices[currency] = Math.round((grouped[time][currency].reduce((partialSum, a) => partialSum + a, 0)) / grouped[time][currency].length); + } + await PricesRepository.$savePrices(parseInt(time, 10), prices); + ++totalInserted; + } + + logger.info(`Inserted ${totalInserted} hourly historical prices into the db`); + } +} + +export default new PriceUpdater(); diff --git a/backend/src/utils/axios-query.ts b/backend/src/utils/axios-query.ts new file mode 100644 index 000000000..8333181f7 --- /dev/null +++ b/backend/src/utils/axios-query.ts @@ -0,0 +1,59 @@ +import axios, { AxiosResponse } from 'axios'; +import { SocksProxyAgent } from 'socks-proxy-agent'; +import backendInfo from '../api/backend-info'; +import config from '../config'; +import logger from '../logger'; +import * as https from 'https'; + +export async function query(path): Promise { + type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; + httpsAgent?: https.Agent; + }; + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; + let retry = 0; + + while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + try { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; + } + + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + + const data: AxiosResponse = await axios.get(path, axiosOptions); + if (data.statusText === 'error' || !data.data) { + throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`); + } + return data.data; + } catch (e) { + logger.err(`Could not connect to ${path}. Reason: ` + (e instanceof Error ? e.message : e)); + retry++; + } + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); + } + return undefined; +} diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 3eb8c5bb0..1032998ef 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -155,13 +155,15 @@ export class BlockComponent implements OnInit, OnDestroy { } }), tap((block: BlockExtended) => { - // Preload previous block summary (execute the http query so the response will be cached) - this.unsubscribeNextBlockSubscriptions(); - setTimeout(() => { - this.nextBlockSubscription = this.apiService.getBlock$(block.previousblockhash).subscribe(); - this.nextBlockTxListSubscription = this.electrsApiService.getBlockTransactions$(block.previousblockhash).subscribe(); - this.nextBlockSummarySubscription = this.apiService.getStrippedBlockTransactions$(block.previousblockhash).subscribe(); - }, 100); + if (block.height > 0) { + // Preload previous block summary (execute the http query so the response will be cached) + this.unsubscribeNextBlockSubscriptions(); + setTimeout(() => { + this.nextBlockSubscription = this.apiService.getBlock$(block.previousblockhash).subscribe(); + this.nextBlockTxListSubscription = this.electrsApiService.getBlockTransactions$(block.previousblockhash).subscribe(); + this.nextBlockSummarySubscription = this.apiService.getStrippedBlockTransactions$(block.previousblockhash).subscribe(); + }, 100); + } this.block = block; this.blockHeight = block.height;