mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-11-20 02:11:49 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/dont-preload-genesis

Browse Source
This commit is contained in:
wiz 2022-06-23 23:43:29 +09:00 committed by GitHub
commit 98b9f007c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 1419 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/src/api/blocks.ts
View File

@ -17,8 +17,6 @@ import { prepareBlock } from '../utils/blocks-utils';
import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
import fiatConversion from './fiat-conversion';
import RatesRepository from '../repositories/RatesRepository';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
@ -461,9 +459,6 @@ class Blocks {
}
}
}
if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) {
await RatesRepository.$saveRate(blockExtended.height, fiatConversion.getConversionRates());
}
if (block.height % 2016 === 0) {
this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;

17
backend/src/api/database-migration.ts
View File

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 20;
private static currentVersion = 21;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -221,6 +221,11 @@ class DatabaseMigration {
if (databaseSchemaVersion < 20 && isBitcoin === true) {
await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
}
if (databaseSchemaVersion < 21) {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
}
} catch (e) {
throw e;
}
@ -526,8 +531,16 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreatePricesTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS prices (
time timestamp NOT NULL,
avg_prices JSON NOT NULL,
PRIMARY KEY (time)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates'];
const allowedTables = ['blocks', 'hashrates', 'prices'];
try {
for (const table of tables) {

2
backend/src/index.ts
View File

@ -27,6 +27,7 @@ import icons from './api/liquid/icons';
import { Common } from './api/common';
import poolsUpdater from './tasks/pools-updater';
import indexer from './indexer';
import priceUpdater from './tasks/price-updater';
class Server {
private wss: WebSocket.Server | undefined;
@ -153,6 +154,7 @@ class Server {
await blocks.$updateBlocks();
await memPool.$updateMempool();
indexer.$run();
priceUpdater.$run();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5;

32
backend/src/repositories/PricesRepository.ts Normal file
View File

@ -0,0 +1,32 @@
import DB from '../database';
import logger from '../logger';
import { Prices } from '../tasks/price-updater';
class PricesRepository {
public async $savePrices(time: number, prices: Prices): Promise<void> {
try {
await DB.query(`INSERT INTO prices(time, avg_prices) VALUE (FROM_UNIXTIME(?), ?)`, [time, JSON.stringify(prices)]);
} catch (e: any) {
logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getOldestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices ORDER BY time LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getLatestPriceTime(): Promise<number> {
const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices ORDER BY time DESC LIMIT 1`);
return oldestRow[0] ? oldestRow[0].time : 0;
}
public async $getPricesTimes(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices`);
return times.map(time => time.time);
}
}
export default new PricesRepository();

21
backend/src/repositories/RatesRepository.ts
View File

@ -1,21 +0,0 @@
import DB from '../database';
import logger from '../logger';
import { IConversionRates } from '../mempool.interfaces';
class RatesRepository {
public async $saveRate(height: number, rates: IConversionRates) {
try {
await DB.query(`INSERT INTO rates(height, bisq_rates) VALUE (?, ?)`, [height, JSON.stringify(rates)]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Rate already exists for block ${height}, ignoring`);
} else {
logger.err(`Cannot save exchange rate into db for block ${height} Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
}
export default new RatesRepository();

16
backend/src/tasks/pools-updater.ts
View File

@ -47,7 +47,7 @@ class PoolsUpdater {
}
if (config.DATABASE.ENABLED === true) {
this.currentSha = await this.getShaFromDb();
this.currentSha = await this.getShaFromDb();
}
logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
@ -70,7 +70,7 @@ class PoolsUpdater {
} catch (e) {
this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
}
}
@ -84,7 +84,7 @@ class PoolsUpdater {
await DB.query('DELETE FROM state where name="pools_json_sha"');
await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
} catch (e) {
logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
}
}
}
@ -97,7 +97,7 @@ class PoolsUpdater {
const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
return (rows.length > 0 ? rows[0].string : undefined);
} catch (e) {
logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
return undefined;
}
}
@ -130,7 +130,7 @@ class PoolsUpdater {
};
timeout: number;
httpsAgent?: https.Agent;
}
};
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const axiosOptions: axiosOptions = {
headers: {
@ -140,7 +140,7 @@ class PoolsUpdater {
};
let retry = 0;
while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try {
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
@ -161,14 +161,14 @@ class PoolsUpdater {
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
const data: AxiosResponse = await axios.get(path, axiosOptions);
if (data.statusText === 'error' || !data.data) {
throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
}
return data.data;
} catch (e) {
logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
retry++;
}
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);

43
backend/src/tasks/price-feeds/bitfinex-api.ts Normal file
View File

@ -0,0 +1,43 @@
import { query } from '../../utils/axios-query';
import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
class BitfinexApi implements PriceFeed {
public name: string = 'Bitfinex';
public currencies: string[] = ['USD', 'EUR', 'GPB', 'JPY'];
public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['last_price'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
if (this.currencies.includes(currency) === false) {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '1h').replace('{CURRENCY}', currency));
const pricesRaw = response ? response : [];
for (const price of pricesRaw as any[]) {
const time = Math.round(price[0] / 1000);
if (priceHistory[time] === undefined) {
priceHistory[time] = priceUpdater.getEmptyPricesObj();
}
priceHistory[time][currency] = price[2];
}
}
return priceHistory;
}
}
export default BitfinexApi;

24
backend/src/tasks/price-feeds/bitflyer-api.ts Normal file
View File

@ -0,0 +1,24 @@
import { query } from '../../utils/axios-query';
import { PriceFeed, PriceHistory } from '../price-updater';
class BitflyerApi implements PriceFeed {
public name: string = 'Bitflyer';
public currencies: string[] = ['USD', 'EUR', 'JPY'];
public url: string = 'https://api.bitflyer.com/v1/ticker?product_code=BTC_';
public urlHist: string = '';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['ltp'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
return [];
}
}
export default BitflyerApi;

42
backend/src/tasks/price-feeds/coinbase-api.ts Normal file
View File

@ -0,0 +1,42 @@
import { query } from '../../utils/axios-query';
import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
class CoinbaseApi implements PriceFeed {
public name: string = 'Coinbase';
public currencies: string[] = ['USD', 'EUR', 'GBP'];
public url: string = 'https://api.coinbase.com/v2/prices/spot?currency=';
public urlHist: string = 'https://api.exchange.coinbase.com/products/BTC-{CURRENCY}/candles?granularity={GRANULARITY}';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['data']['amount'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
if (this.currencies.includes(currency) === false) {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
const pricesRaw = response ? response : [];
for (const price of pricesRaw as any[]) {
if (priceHistory[price[0]] === undefined) {
priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
}
priceHistory[price[0]][currency] = price[4];
}
}
return priceHistory;
}
}
export default CoinbaseApi;

43
backend/src/tasks/price-feeds/ftx-api.ts Normal file
View File

@ -0,0 +1,43 @@
import { query } from '../../utils/axios-query';
import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
class FtxApi implements PriceFeed {
public name: string = 'FTX';
public currencies: string[] = ['USD', 'BRZ', 'EUR', 'JPY', 'AUD'];
public url: string = 'https://ftx.com/api/markets/BTC/';
public urlHist: string = 'https://ftx.com/api/markets/BTC/{CURRENCY}/candles?resolution={GRANULARITY}';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['result']['last'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
if (this.currencies.includes(currency) === false) {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
const pricesRaw = response ? response['result'] : [];
for (const price of pricesRaw as any[]) {
const time = Math.round(price['time'] / 1000);
if (priceHistory[time] === undefined) {
priceHistory[time] = priceUpdater.getEmptyPricesObj();
}
priceHistory[time][currency] = price['close'];
}
}
return priceHistory;
}
}
export default FtxApi;

43
backend/src/tasks/price-feeds/gemini-api.ts Normal file
View File

@ -0,0 +1,43 @@
import { query } from '../../utils/axios-query';
import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
class GeminiApi implements PriceFeed {
public name: string = 'Gemini';
public currencies: string[] = ['USD', 'EUR', 'GBP', 'SGD'];
public url: string = 'https://api.gemini.com/v1/pubticker/BTC';
public urlHist: string = 'https://api.gemini.com/v2/candles/BTC{CURRENCY}/{GRANULARITY}';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['last'], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
if (this.currencies.includes(currency) === false) {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '1hr').replace('{CURRENCY}', currency));
const pricesRaw = response ? response : [];
for (const price of pricesRaw as any[]) {
const time = Math.round(price[0] / 1000);
if (priceHistory[time] === undefined) {
priceHistory[time] = priceUpdater.getEmptyPricesObj();
}
priceHistory[time][currency] = price[4];
}
}
return priceHistory;
}
}
export default GeminiApi;

95
backend/src/tasks/price-feeds/kraken-api.ts Normal file
View File

@ -0,0 +1,95 @@
import logger from '../../logger';
import PricesRepository from '../../repositories/PricesRepository';
import { query } from '../../utils/axios-query';
import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
class KrakenApi implements PriceFeed {
public name: string = 'Kraken';
public currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
public url: string = 'https://api.kraken.com/0/public/Ticker?pair=XBT';
public urlHist: string = 'https://api.kraken.com/0/public/OHLC?interval={GRANULARITY}&pair=XBT';
constructor() {
}
private getTicker(currency) {
let ticker = `XXBTZ${currency}`;
if (['CHF', 'AUD'].includes(currency)) {
ticker = `XBT${currency}`;
}
return ticker;
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
}
public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
const priceHistory: PriceHistory = {};
for (const currency of currencies) {
if (this.currencies.includes(currency) === false) {
continue;
}
const response = await query(this.urlHist.replace('{GRANULARITY}', '60') + currency);
const pricesRaw = response ? response['result'][this.getTicker(currency)] : [];
for (const price of pricesRaw) {
if (priceHistory[price[0]] === undefined) {
priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
}
priceHistory[price[0]][currency] = price[4];
}
}
return priceHistory;
}
/**
* Fetch weekly price and save it into the database
*/
public async $insertHistoricalPrice(): Promise<void> {
const existingPriceTimes = await PricesRepository.$getPricesTimes();
// EUR weekly price history goes back to timestamp 1378339200 (September 5, 2013)
// USD weekly price history goes back to timestamp 1380758400 (October 3, 2013)
// GBP weekly price history goes back to timestamp 1415232000 (November 6, 2014)
// JPY weekly price history goes back to timestamp 1415232000 (November 6, 2014)
// CAD weekly price history goes back to timestamp 1436400000 (July 9, 2015)
// CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019)
// AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020)
const priceHistory: any = {}; // map: timestamp -> Prices
for (const currency of this.currencies) {
const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency);
const priceHistoryRaw = response ? response['result'][this.getTicker(currency)] : [];
for (const price of priceHistoryRaw) {
if (existingPriceTimes.includes(parseInt(price[0]))) {
continue;
}
// prices[0] = kraken price timestamp
// prices[4] = closing price
if (priceHistory[price[0]] === undefined) {
priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
}
priceHistory[price[0]][currency] = price[4];
}
}
for (const time in priceHistory) {
await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]);
}
if (Object.keys(priceHistory).length > 0) {
logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
}
}
}
export default KrakenApi;

762
backend/src/tasks/price-feeds/mtgox-weekly.json Normal file
View File

@ -0,0 +1,762 @@
[
{
"ct": 1279497600,
"c": "0.08584"
},
{
"ct": 1280102400,
"c": "0.0505"
},
{
"ct": 1280707200,
"c": "0.0611"
},
{
"ct": 1281312000,
"c": "0.0609"
},
{
"ct": 1281916800,
"c": "0.06529"
},
{
"ct": 1282521600,
"c": "0.066"
},
{
"ct": 1283126400,
"c": "0.064"
},
{
"ct": 1283731200,
"c": "0.06165"
},
{
"ct": 1284336000,
"c": "0.0615"
},
{
"ct": 1284940800,
"c": "0.0627"
},
{
"ct": 1285545600,
"c": "0.0622"
},
{
"ct": 1286150400,
"c": "0.06111"
},
{
"ct": 1286755200,
"c": "0.0965"
},
{
"ct": 1287360000,
"c": "0.102"
},
{
"ct": 1287964800,
"c": "0.11501"
},
{
"ct": 1288569600,
"c": "0.1925"
},
{
"ct": 1289174400,
"c": "0.34"
},
{
"ct": 1289779200,
"c": "0.27904"
},
{
"ct": 1290384000,
"c": "0.27675"
},
{
"ct": 1290988800,
"c": "0.27"
},
{
"ct": 1291593600,
"c": "0.19"
},
{
"ct": 1292198400,
"c": "0.2189"
},
{
"ct": 1292803200,
"c": "0.2401"
},
{
"ct": 1293408000,
"c": "0.263"
},
{
"ct": 1294012800,
"c": "0.29997"
},
{
"ct": 1294617600,
"c": "0.323"
},
{
"ct": 1295222400,
"c": "0.38679"
},
{
"ct": 1295827200,
"c": "0.4424"
},
{
"ct": 1296432000,
"c": "0.4799"
},
{
"ct": 1297036800,
"c": "0.8968"
},
{
"ct": 1297641600,
"c": "1.05"
},
{
"ct": 1298246400,
"c": "0.865"
},
{
"ct": 1298851200,
"c": "0.89"
},
{
"ct": 1299456000,
"c": "0.8999"
},
{
"ct": 1300060800,
"c": "0.89249"
},
{
"ct": 1300665600,
"c": "0.75218"
},
{
"ct": 1301270400,
"c": "0.82754"
},
{
"ct": 1301875200,
"c": "0.779"
},
{
"ct": 1302480000,
"c": "0.7369"
},
{
"ct": 1303084800,
"c": "1.1123"
},
{
"ct": 1303689600,
"c": "1.6311"
},
{
"ct": 1304294400,
"c": "3.03311"
},
{
"ct": 1304899200,
"c": "3.8659"
},
{
"ct": 1305504000,
"c": "6.98701"
},
{
"ct": 1306108800,
"c": "6.6901"
},
{
"ct": 1306713600,
"c": "8.4"
},
{
"ct": 1307318400,
"c": "16.7"
},
{
"ct": 1307923200,
"c": "18.5464"
},
{
"ct": 1308528000,
"c": "17.51"
},
{
"ct": 1309132800,
"c": "16.45001"
},
{
"ct": 1309737600,
"c": "15.44049"
},
{
"ct": 1310342400,
"c": "14.879"
},
{
"ct": 1310947200,
"c": "13.16"
},
{
"ct": 1311552000,
"c": "13.98001"
},
{
"ct": 1312156800,
"c": "13.35"
},
{
"ct": 1312761600,
"c": "7.9"
},
{
"ct": 1313366400,
"c": "10.7957"
},
{
"ct": 1313971200,
"c": "11.31125"
},
{
"ct": 1314576000,
"c": "9.07011"
},
{
"ct": 1315180800,
"c": "8.17798"
},
{
"ct": 1315785600,
"c": "5.86436"
},
{
"ct": 1316390400,
"c": "5.2"
},
{
"ct": 1316995200,
"c": "5.33"
},
{
"ct": 1317600000,
"c": "5.02701"
},
{
"ct": 1318204800,
"c": "4.10288"
},
{
"ct": 1318809600,
"c": "3.5574"
},
{
"ct": 1319414400,
"c": "3.12657"
},
{
"ct": 1320019200,
"c": "3.27"
},
{
"ct": 1320624000,
"c": "2.95959"
},
{
"ct": 1321228800,
"c": "2.99626"
},
{
"ct": 1321833600,
"c": "2.2"
},
{
"ct": 1322438400,
"c": "2.47991"
},
{
"ct": 1323043200,
"c": "2.82809"
},
{
"ct": 1323648000,
"c": "3.2511"
},
{
"ct": 1324252800,
"c": "3.193"
},
{
"ct": 1324857600,
"c": "4.225"
},
{
"ct": 1325462400,
"c": "5.26766"
},
{
"ct": 1326067200,
"c": "7.11358"
},
{
"ct": 1326672000,
"c": "7.00177"
},
{
"ct": 1327276800,
"c": "6.3097"
},
{
"ct": 1327881600,
"c": "5.38191"
},
{
"ct": 1328486400,
"c": "5.68881"
},
{
"ct": 1329091200,
"c": "5.51468"
},
{
"ct": 1329696000,
"c": "4.38669"
},
{
"ct": 1330300800,
"c": "4.922"
},
{
"ct": 1330905600,
"c": "4.8201"
},
{
"ct": 1331510400,
"c": "4.90901"
},
{
"ct": 1332115200,
"c": "5.27943"
},
{
"ct": 1332720000,
"c": "4.55001"
},
{
"ct": 1333324800,
"c": "4.81922"
},
{
"ct": 1333929600,
"c": "4.79253"
},
{
"ct": 1334534400,
"c": "4.96892"
},
{
"ct": 1335139200,
"c": "5.20352"
},
{
"ct": 1335744000,
"c": "4.90441"
},
{
"ct": 1336348800,
"c": "5.04991"
},
{
"ct": 1336953600,
"c": "4.92996"
},
{
"ct": 1337558400,
"c": "5.09002"
},
{
"ct": 1338163200,
"c": "5.13896"
},
{
"ct": 1338768000,
"c": "5.2051"
},
{
"ct": 1339372800,
"c": "5.46829"
},
{
"ct": 1339977600,
"c": "6.16382"
},
{
"ct": 1340582400,
"c": "6.35002"
},
{
"ct": 1341187200,
"c": "6.62898"
},
{
"ct": 1341792000,
"c": "6.79898"
},
{
"ct": 1342396800,
"c": "7.62101"
},
{
"ct": 1343001600,
"c": "8.4096"
},
{
"ct": 1343606400,
"c": "8.71027"
},
{
"ct": 1344211200,
"c": "10.86998"
},
{
"ct": 1344816000,
"c": "11.6239"
},
{
"ct": 1345420800,
"c": "7.98"
},
{
"ct": 1346025600,
"c": "10.61"
},
{
"ct": 1346630400,
"c": "10.2041"
},
{
"ct": 1347235200,
"c": "11.02"
},
{
"ct": 1347840000,
"c": "11.87"
},
{
"ct": 1348444800,
"c": "12.19331"
},
{
"ct": 1349049600,
"c": "12.4"
},
{
"ct": 1349654400,
"c": "11.8034"
},
{
"ct": 1350259200,
"c": "11.7389"
},
{
"ct": 1350864000,
"c": "11.63107"
},
{
"ct": 1351468800,
"c": "10.69998"
},
{
"ct": 1352073600,
"c": "10.80011"
},
{
"ct": 1352678400,
"c": "10.84692"
},
{
"ct": 1353283200,
"c": "11.65961"
},
{
"ct": 1353888000,
"c": "12.4821"
},
{
"ct": 1354492800,
"c": "12.50003"
},
{
"ct": 1355097600,
"c": "13.388"
},
{
"ct": 1355702400,
"c": "13.30002"
},
{
"ct": 1356307200,
"c": "13.31202"
},
{
"ct": 1356912000,
"c": "13.45001"
},
{
"ct": 1357516800,
"c": "13.5199"
},
{
"ct": 1358121600,
"c": "14.11601"
},
{
"ct": 1358726400,
"c": "15.7"
},
{
"ct": 1359331200,
"c": "17.95"
},
{
"ct": 1359936000,
"c": "20.59"
},
{
"ct": 1360540800,
"c": "23.96975"
},
{
"ct": 1361145600,
"c": "26.8146"
},
{
"ct": 1361750400,
"c": "29.88999"
},
{
"ct": 1362355200,
"c": "34.49999"
},
{
"ct": 1362960000,
"c": "46"
},
{
"ct": 1363564800,
"c": "47.4"
},
{
"ct": 1364169600,
"c": "71.93"
},
{
"ct": 1364774400,
"c": "93.03001"
},
{
"ct": 1365379200,
"c": "162.30102"
},
{
"ct": 1365984000,
"c": "89.99999"
},
{
"ct": 1366588800,
"c": "119.2"
},
{
"ct": 1367193600,
"c": "134.44444"
},
{
"ct": 1367798400,
"c": "115.98"
},
{
"ct": 1368403200,
"c": "114.82002"
},
{
"ct": 1369008000,
"c": "122.49999"
},
{
"ct": 1369612800,
"c": "133.5"
},
{
"ct": 1370217600,
"c": "122.5"
},
{
"ct": 1370822400,
"c": "100.43743"
},
{
"ct": 1371427200,
"c": "99.9"
},
{
"ct": 1372032000,
"c": "107.90001"
},
{
"ct": 1372636800,
"c": "97.51"
},
{
"ct": 1373241600,
"c": "76.5"
},
{
"ct": 1373846400,
"c": "94.41986"
},
{
"ct": 1374451200,
"c": "91.998"
},
{
"ct": 1375056000,
"c": "98.78008"
},
{
"ct": 1375660800,
"c": "105.12"
},
{
"ct": 1376265600,
"c": "105"
},
{
"ct": 1376870400,
"c": "113.38"
},
{
"ct": 1377475200,
"c": "122.11102"
},
{
"ct": 1378080000,
"c": "146.01003"
},
{
"ct": 1378684800,
"c": "126.31501"
},
{
"ct": 1379289600,
"c": "138.3002"
},
{
"ct": 1379894400,
"c": "134.00001"
},
{
"ct": 1380499200,
"c": "143.88402"
},
{
"ct": 1381104000,
"c": "137.8"
},
{
"ct": 1381708800,
"c": "147.53"
},
{
"ct": 1382313600,
"c": "186.1"
},
{
"ct": 1382918400,
"c": "207.0001"
},
{
"ct": 1383523200,
"c": "224.01001"
},
{
"ct": 1384128000,
"c": "336.33101"
},
{
"ct": 1384732800,
"c": "528"
},
{
"ct": 1385337600,
"c": "795"
},
{
"ct": 1385942400,
"c": "1004.42392"
},
{
"ct": 1386547200,
"c": "804.5"
},
{
"ct": 1387152000,
"c": "919.985"
},
{
"ct": 1387756800,
"c": "639.48"
},
{
"ct": 1388361600,
"c": "786.98"
},
{
"ct": 1388966400,
"c": "1015"
},
{
"ct": 1389571200,
"c": "940"
},
{
"ct": 1390176000,
"c": "954.995"
},
{
"ct": 1390780800,
"c": "1007.98999"
},
{
"ct": 1391385600,
"c": "954"
},
{
"ct": 1391990400,
"c": "659.49776"
},
{
"ct": 1392595200,
"c": "299.702"
},
{
"ct": 1393200000,
"c": "310.00001"
},
{
"ct": 1393804800,
"c": "135"
}
]

251
backend/src/tasks/price-updater.ts Normal file
View File

@ -0,0 +1,251 @@
import * as fs from 'fs';
import config from '../config';
import logger from '../logger';
import PricesRepository from '../repositories/PricesRepository';
import BitfinexApi from './price-feeds/bitfinex-api';
import BitflyerApi from './price-feeds/bitflyer-api';
import CoinbaseApi from './price-feeds/coinbase-api';
import FtxApi from './price-feeds/ftx-api';
import GeminiApi from './price-feeds/gemini-api';
import KrakenApi from './price-feeds/kraken-api';
export interface PriceFeed {
name: string;
url: string;
urlHist: string;
currencies: string[];
$fetchPrice(currency): Promise<number>;
$fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory>;
}
export interface PriceHistory {
[timestamp: number]: Prices;
}
export interface Prices {
USD: number;
EUR: number;
GBP: number;
CAD: number;
CHF: number;
AUD: number;
JPY: number;
}
class PriceUpdater {
historyInserted: boolean = false;
lastRun: number = 0;
lastHistoricalRun: number = 0;
running: boolean = false;
feeds: PriceFeed[] = [];
currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
latestPrices: Prices;
constructor() {
this.latestPrices = this.getEmptyPricesObj();
this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
this.feeds.push(new FtxApi());
this.feeds.push(new KrakenApi());
this.feeds.push(new CoinbaseApi());
this.feeds.push(new BitfinexApi());
this.feeds.push(new GeminiApi());
}
public getEmptyPricesObj(): Prices {
return {
USD: -1,
EUR: -1,
GBP: -1,
CAD: -1,
CHF: -1,
AUD: -1,
JPY: -1,
};
}
public async $run(): Promise<void> {
if (this.running === true) {
return;
}
this.running = true;
if ((Math.round(new Date().getTime() / 1000) - this.lastHistoricalRun) > 3600 * 24) {
// Once a day, look for missing prices (could happen due to network connectivity issues)
this.historyInserted = false;
}
try {
if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
await this.$insertHistoricalPrices();
} else {
await this.$updatePrice();
}
} catch (e) {
logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`);
}
this.running = false;
}
/**
* Fetch last BTC price from exchanges, average them, and save it in the database once every hour
*/
private async $updatePrice(): Promise<void> {
if (this.lastRun === 0 && config.DATABASE.ENABLED === true) {
this.lastRun = await PricesRepository.$getLatestPriceTime();
}
if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) {
// Refresh only once every hour
return;
}
const previousRun = this.lastRun;
this.lastRun = new Date().getTime() / 1000;
for (const currency of this.currencies) {
let prices: number[] = [];
for (const feed of this.feeds) {
// Fetch prices from API which supports `currency`
if (feed.currencies.includes(currency)) {
try {
const price = await feed.$fetchPrice(currency);
if (price > 0) {
prices.push(price);
}
logger.debug(`${feed.name} BTC/${currency} price: ${price}`);
} catch (e) {
logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`);
}
}
}
if (prices.length === 1) {
logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`);
}
// Compute average price, non weighted
prices = prices.filter(price => price > 0);
this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
}
logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
if (config.DATABASE.ENABLED === true) {
// Save everything in db
try {
const p = 60 * 60 * 1000; // milliseconds in an hour
const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
} catch (e) {
this.lastRun = previousRun + 5 * 60;
logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
}
}
this.lastRun = new Date().getTime() / 1000;
}
/**
* Called once by the database migration to initialize historical prices data (weekly)
* We use MtGox weekly price from July 19, 2010 to September 30, 2013
* We use Kraken weekly price from October 3, 2013 up to last month
* We use Kraken hourly price for the past month
*/
private async $insertHistoricalPrices(): Promise<void> {
const existingPriceTimes = await PricesRepository.$getPricesTimes();
// Insert MtGox weekly prices
const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
const prices = this.getEmptyPricesObj();
let insertedCount: number = 0;
for (const price of pricesJson) {
if (existingPriceTimes.includes(price['ct'])) {
continue;
}
// From 1380758400 we will use Kraken price as it follows closely MtGox, but was not affected as much
// by the MtGox exchange collapse a few months later
if (price['ct'] > 1380758400) {
break;
}
prices.USD = price['c'];
await PricesRepository.$savePrices(price['ct'], prices);
++insertedCount;
}
if (insertedCount > 0) {
logger.info(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
}
// Insert Kraken weekly prices
await new KrakenApi().$insertHistoricalPrice();
// Insert missing recent hourly prices
await this.$insertMissingRecentPrices();
this.historyInserted = true;
this.lastHistoricalRun = new Date().getTime();
}
/**
* Find missing hourly prices and insert them in the database
* It has a limited backward range and it depends on which API are available
*/
private async $insertMissingRecentPrices(): Promise<void> {
const existingPriceTimes = await PricesRepository.$getPricesTimes();
logger.info(`Fetching hourly price history from exchanges and saving missing ones into the database, this may take a while`);
const historicalPrices: PriceHistory[] = [];
// Fetch all historical hourly prices
for (const feed of this.feeds) {
try {
historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies));
} catch (e) {
logger.info(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
}
}
// Group them by timestamp and currency, for example
// grouped[123456789]['USD'] = [1, 2, 3, 4];
let grouped: Object = {};
for (const historicalEntry of historicalPrices) {
for (const time in historicalEntry) {
if (existingPriceTimes.includes(parseInt(time, 10))) {
continue;
}
if (grouped[time] == undefined) {
grouped[time] = {
USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: []
}
}
for (const currency of this.currencies) {
const price = historicalEntry[time][currency];
if (price > 0) {
grouped[time][currency].push(parseInt(price, 10));
}
}
}
}
// Average prices and insert everything into the db
let totalInserted = 0;
for (const time in grouped) {
const prices: Prices = this.getEmptyPricesObj();
for (const currency in grouped[time]) {
prices[currency] = Math.round((grouped[time][currency].reduce((partialSum, a) => partialSum + a, 0)) / grouped[time][currency].length);
}
await PricesRepository.$savePrices(parseInt(time, 10), prices);
++totalInserted;
}
logger.info(`Inserted ${totalInserted} hourly historical prices into the db`);
}
}
export default new PriceUpdater();

59
backend/src/utils/axios-query.ts Normal file
View File

@ -0,0 +1,59 @@
import axios, { AxiosResponse } from 'axios';
import { SocksProxyAgent } from 'socks-proxy-agent';
import backendInfo from '../api/backend-info';
import config from '../config';
import logger from '../logger';
import * as https from 'https';
export async function query(path): Promise<object | undefined> {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpsAgent?: https.Agent;
};
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0;
while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try {
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
}
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
const data: AxiosResponse = await axios.get(path, axiosOptions);
if (data.statusText === 'error' || !data.data) {
throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
}
return data.data;
} catch (e) {
logger.err(`Could not connect to ${path}. Reason: ` + (e instanceof Error ? e.message : e));
retry++;
}
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
}
return undefined;
}