From a11135f358d5eb2094e39deabbc8eda5843098d7 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 15 Sep 2020 03:11:52 +0700 Subject: [PATCH] * /volumes API * /ticker API --- backend/mempool-config.sample.json | 4 +- backend/src/api/bisq/bisq.ts | 16 +- backend/src/api/bisq/interfaces.ts | 14 +- backend/src/api/bisq/markets-api.ts | 272 +++++++++++++++++++++------- backend/src/api/bisq/markets.ts | 2 +- backend/src/routes.ts | 24 ++- production/mempool-config.bisq.json | 2 +- 7 files changed, 246 insertions(+), 88 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 190fd7164..9f7b773fb 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -15,9 +15,9 @@ "TX_PER_SECOND_SPAN_SECONDS": 150, "ELECTRS_API_URL": "http://localhost:50001", "BISQ_ENABLED": false, + "BISQ_BLOCKS_DATA_PATH": "/bisq/seednode-data/btc_mainnet/db/json", "BISQ_MARKET_ENABLED": false, - "BSQ_BLOCKS_DATA_PATH": "/bisq/data", - "BSQ_MARKETS_DATA_PATH": "/bisq-folder/Bisq", + "BISQ_MARKETS_DATA_PATH": "/bisq/seednode-data/btc_mainnet/db", "SSL": false, "SSL_CERT_FILE_PATH": "/etc/letsencrypt/live/mysite/fullchain.pem", "SSL_KEY_FILE_PATH": "/etc/letsencrypt/live/mysite/privkey.pem" diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 0f4cdee81..5c5101890 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -72,8 +72,8 @@ class Bisq { } private checkForBisqDataFolder() { - if (!fs.existsSync(config.BSQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { - console.log(config.BSQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`); + if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { + console.log(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`); return process.exit(1); } } @@ -83,7 +83,7 @@ class Bisq { this.topDirectoryWatcher.close(); } let fsWait: NodeJS.Timeout | null = null; - this.topDirectoryWatcher = fs.watch(config.BSQ_BLOCKS_DATA_PATH, () => { + this.topDirectoryWatcher = fs.watch(config.BISQ_BLOCKS_DATA_PATH, () => { if (fsWait) { clearTimeout(fsWait); } @@ -105,13 +105,13 @@ class Bisq { if (this.subdirectoryWatcher) { this.subdirectoryWatcher.close(); } - if (!fs.existsSync(config.BSQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { - console.log(config.BSQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`); + if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { + console.log(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`); setTimeout(() => this.startSubDirectoryWatcher(), 180000); return; } let fsWait: NodeJS.Timeout | null = null; - this.subdirectoryWatcher = fs.watch(config.BSQ_BLOCKS_DATA_PATH + '/all', () => { + this.subdirectoryWatcher = fs.watch(config.BISQ_BLOCKS_DATA_PATH + '/all', () => { if (fsWait) { clearTimeout(fsWait); } @@ -243,10 +243,10 @@ class Bisq { private loadData(): Promise { return new Promise((resolve, reject) => { - if (!fs.existsSync(config.BSQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { + if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`); } - fs.readFile(config.BSQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => { + fs.readFile(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => { if (err) { reject(err); } diff --git a/backend/src/api/bisq/interfaces.ts b/backend/src/api/bisq/interfaces.ts index 1eb890fc5..f7cfaba43 100644 --- a/backend/src/api/bisq/interfaces.ts +++ b/backend/src/api/bisq/interfaces.ts @@ -234,8 +234,8 @@ export interface TradesData { export interface MarketVolume { period_start: number; - volume: string; num_trades: number; + volume: string; } export interface MarketsApiError { @@ -244,3 +244,15 @@ export interface MarketsApiError { } export type Interval = 'minute' | 'half_hour' | 'hour' | 'half_day' | 'day' | 'week' | 'month' | 'year' | 'auto'; + +export interface SummarizedIntervals { [market: string]: SummarizedInterval; } +export interface SummarizedInterval { + 'period_start': number; + 'open': number; + 'close': number; + 'high': number; + 'low': number; + 'avg': number; + 'volume_right': number; + 'volume_left': number; +} diff --git a/backend/src/api/bisq/markets-api.ts b/backend/src/api/bisq/markets-api.ts index 910fc2d0b..64e027c02 100644 --- a/backend/src/api/bisq/markets-api.ts +++ b/backend/src/api/bisq/markets-api.ts @@ -1,5 +1,5 @@ import { Currencies, OffsersData, TradesData, Depth, Currency, Interval, HighLowOpenClose, - Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers } from './interfaces'; + Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces'; import * as datetime from 'locutus/php/datetime'; @@ -8,6 +8,8 @@ class BisqMarketsApi { private fiatCurrencyData: Currency[] = []; private offersData: OffsersData[] = []; private tradesData: TradesData[] = []; + private fiatCurrenciesIndexed: { [code: string]: true } = {}; + private tradeDataByMarket: { [market: string]: TradesData[] } = {}; constructor() { } @@ -17,10 +19,18 @@ class BisqMarketsApi { this.offersData = offers; this.tradesData = trades; - this.fiatCurrencyData.forEach((currency) => currency._type = 'fiat'); + // Handle data for smarter memory caching + this.fiatCurrencyData.forEach((currency) => { + currency._type = 'fiat'; + this.fiatCurrenciesIndexed[currency.code] = true; + }); this.cryptoCurrencyData.forEach((currency) => currency._type = 'crypto'); this.tradesData.forEach((trade) => { trade._market = trade.currencyPair.toLowerCase().replace('/', '_'); + if (!this.tradeDataByMarket[trade._market]) { + this.tradeDataByMarket[trade._market] = []; + } + this.tradeDataByMarket[trade._market].push(trade); }); } @@ -153,7 +163,7 @@ class BisqMarketsApi { sort: 'asc' | 'desc' = 'desc', ): BisqTrade[] { limit = Math.min(limit, 2000); - const _market = market === 'all' ? null : market; + const _market = market === 'all' ? undefined : market; if (!timestamp_from) { timestamp_from = new Date('2016-01-01').getTime() / 1000; @@ -188,18 +198,149 @@ class BisqMarketsApi { } getVolumes( - timestamp_from: number, - timestamp_to: number, - interval: Interval, market?: string, - ): MarketVolume[] { - return []; + timestamp_from?: number, + timestamp_to?: number, + interval: Interval = 'auto', + milliseconds?: boolean, + ): MarketVolume[] { + if (milliseconds) { + timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from; + timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to; + } + if (!timestamp_from) { + timestamp_from = new Date('2016-01-01').getTime() / 1000; + } + if (!timestamp_to) { + timestamp_to = new Date().getTime() / 1000; + } + + const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from, + undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER); + + if (interval === 'auto') { + const range = timestamp_to - timestamp_from; + interval = this.getIntervalFromRange(range); + } + + const intervals: any = {}; + const marketVolumes: MarketVolume[] = []; + + for (const trade of trades) { + const traded_at = trade['tradeDate'] / 1000; + const interval_start = this.intervalStart(traded_at, interval); + + if (!intervals[interval_start]) { + intervals[interval_start] = { + 'volume': 0, + 'num_trades': 0, + }; + } + + const period = intervals[interval_start]; + period['period_start'] = interval_start; + period['volume'] += this.fiatCurrenciesIndexed[trade.currency] ? trade._tradeAmount : trade._tradeVolume; + period['num_trades']++; + } + + for (const p in intervals) { + if (intervals.hasOwnProperty(p)) { + const period = intervals[p]; + marketVolumes.push({ + period_start: period['period_start'], + num_trades: period['num_trades'], + volume: this.intToBtc(period['volume']), + }); + } + } + + return intervals; } getTicker( market?: string, - ): Tickers { - return {}; + ): Tickers | Ticker | null { + if (market) { + return this.getTickerFromMarket(market); + } + + const allMarkets = this.getMarkets(); + const tickers = {}; + for (const m in allMarkets) { + if (allMarkets.hasOwnProperty(m)) { + tickers[allMarkets[m].pair] = this.getTickerFromMarket(allMarkets[m].pair); + } + } + + return tickers; + } + + getTickerFromMarket(market: string): Ticker | null { + let ticker: Ticker; + const timestamp_from = datetime.strtotime('-24 hour'); + const timestamp_to = new Date().getTime() / 1000; + const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from, + undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER); + + const periods: SummarizedInterval[] = Object.values(this.getTradesSummarized(trades, timestamp_from)); + + const allCurrencies = this.getCurrencies(); + const currencyRight = allCurrencies[market.split('_')[1].toUpperCase()]; + + if (periods[0]) { + ticker = { + 'last': this.intToBtc(periods[0].close), + 'high': this.intToBtc(periods[0].high), + 'low': this.intToBtc(periods[0].low), + 'volume_left': this.intToBtc(periods[0].volume_left), + 'volume_right': this.intToBtc(periods[0].volume_right), + 'buy': null, + 'sell': null, + }; + } else { + const lastTrade = this.tradeDataByMarket[market]; + if (!lastTrade) { + return null; + } + const tradePrice = lastTrade[0].primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision); + + const lastTradePrice = this.intToBtc(tradePrice); + ticker = { + 'last': lastTradePrice, + 'high': lastTradePrice, + 'low': lastTradePrice, + 'volume_left': '0', + 'volume_right': '0', + 'buy': null, + 'sell': null, + }; + } + + const timestampFromMilli = timestamp_from * 1000; + const timestampToMilli = timestamp_to * 1000; + + const currencyPair = market.replace('_', '/').toUpperCase(); + const offersData = this.offersData.slice().sort((a, b) => a.price - b.price); + + const buy = offersData.find((offer) => offer.currencyPair === currencyPair + && offer.primaryMarketDirection === 'BUY' + && offer.date >= timestampFromMilli + && offer.date <= timestampToMilli + ); + const sell = offersData.find((offer) => offer.currencyPair === currencyPair + && offer.primaryMarketDirection === 'SELL' + && offer.date >= timestampFromMilli + && offer.date <= timestampToMilli + ); + + if (buy) { + ticker.buy = this.intToBtc(buy.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision)); + } + if (sell) { + ticker.sell = this.intToBtc(sell.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision)); + } + + return ticker; } getHloc( @@ -220,53 +361,73 @@ class BisqMarketsApi { timestamp_to = new Date().getTime() / 1000; } - const range = timestamp_to - timestamp_from; - const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from, undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER); if (interval === 'auto') { - // two days range loads minute data - if (range <= 3600) { - // up to one hour range loads minutely data - interval = 'minute'; - } else if (range <= 1 * 24 * 3600) { - // up to one day range loads half-hourly data - interval = 'half_hour'; - } else if (range <= 3 * 24 * 3600) { - // up to 3 day range loads hourly data - interval = 'hour'; - } else if (range <= 7 * 24 * 3600) { - // up to 7 day range loads half-daily data - interval = 'half_day'; - } else if (range <= 60 * 24 * 3600) { - // up to 2 month range loads daily data - interval = 'day'; - } else if (range <= 12 * 31 * 24 * 3600) { - // up to one year range loads weekly data - interval = 'week'; - } else if (range <= 12 * 31 * 24 * 3600) { - // up to 5 year range loads monthly data - interval = 'month'; - } else { - // greater range loads yearly data - interval = 'year'; - } + const range = timestamp_to - timestamp_from; + interval = this.getIntervalFromRange(range); } - const hlocs = this.getTradesSummarized(trades, timestamp_from, interval); + const intervals = this.getTradesSummarized(trades, timestamp_from, interval); - return hlocs; + const hloc: HighLowOpenClose[] = []; + + for (const p in intervals) { + if (intervals.hasOwnProperty(p)) { + const period = intervals[p]; + hloc.push({ + period_start: period['period_start'], + open: this.intToBtc(period['open']), + close: this.intToBtc(period['close']), + high: this.intToBtc(period['high']), + low: this.intToBtc(period['low']), + avg: this.intToBtc(period['avg']), + volume_right: this.intToBtc(period['volume_right']), + volume_left: this.intToBtc(period['volume_left']), + }); + } + } + + return hloc; } - private getTradesSummarized(trades: TradesData[], timestamp_from, interval: string): HighLowOpenClose[] { + private getIntervalFromRange(range: number): Interval { + // two days range loads minute data + if (range <= 3600) { + // up to one hour range loads minutely data + return 'minute'; + } else if (range <= 1 * 24 * 3600) { + // up to one day range loads half-hourly data + return 'half_hour'; + } else if (range <= 3 * 24 * 3600) { + // up to 3 day range loads hourly data + return 'hour'; + } else if (range <= 7 * 24 * 3600) { + // up to 7 day range loads half-daily data + return 'half_day'; + } else if (range <= 60 * 24 * 3600) { + // up to 2 month range loads daily data + return 'day'; + } else if (range <= 12 * 31 * 24 * 3600) { + // up to one year range loads weekly data + return 'week'; + } else if (range <= 12 * 31 * 24 * 3600) { + // up to 5 year range loads monthly data + return 'month'; + } else { + // greater range loads yearly data + return 'year'; + } + } + + private getTradesSummarized(trades: TradesData[], timestamp_from: number, interval?: string): SummarizedIntervals { const intervals: any = {}; const intervals_prices: any = {}; - const one_period = false; for (const trade of trades) { const traded_at = trade.tradeDate / 1000; - const interval_start = one_period ? timestamp_from : this.intervalStart(traded_at, interval); + const interval_start = !interval ? timestamp_from : this.intervalStart(traded_at, interval); if (!intervals[interval_start]) { intervals[interval_start] = { @@ -306,30 +467,11 @@ class BisqMarketsApi { period['volume_right'] += trade._tradeVolume; } } - - const hloc: HighLowOpenClose[] = []; - - for (const p in intervals) { - if (intervals.hasOwnProperty(p)) { - const period = intervals[p]; - hloc.push({ - period_start: period['period_start'], - open: this.intToBtc(period['open']), - close: this.intToBtc(period['close']), - high: this.intToBtc(period['high']), - low: this.intToBtc(period['low']), - avg: this.intToBtc(period['avg']), - volume_right: this.intToBtc(period['volume_right']), - volume_left: this.intToBtc(period['volume_left']), - }); - } - } - - return hloc; + return intervals; } private getTradesByCriteria( - market: string | null, + market: string | undefined, timestamp_to: number, timestamp_from: number, trade_id_to: string | undefined, @@ -422,7 +564,7 @@ class BisqMarketsApi { return matches; } - private intervalStart(ts: number, interval: string) { + private intervalStart(ts: number, interval: string): number { switch (interval) { case 'minute': return (ts - (ts % 60)); diff --git a/backend/src/api/bisq/markets.ts b/backend/src/api/bisq/markets.ts index d9902a00b..880d930ae 100644 --- a/backend/src/api/bisq/markets.ts +++ b/backend/src/api/bisq/markets.ts @@ -4,7 +4,7 @@ import { OffsersData, TradesData, Currency } from './interfaces'; import bisqMarket from './markets-api'; class Bisq { - private static MARKET_JSON_PATH = config.BSQ_MARKETS_DATA_PATH + '/btc_mainnet/db'; + private static MARKET_JSON_PATH = config.BISQ_MARKETS_DATA_PATH + '/btc_mainnet/db'; private static MARKET_JSON_FILE_PATHS = { cryptoCurrency: '/crypto_currency_list.json', fiatCurrency: '/fiat_currency_list.json', diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 0aaea5212..f4939f52c 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -175,7 +175,7 @@ class Routes { const p = this.parseRequestParameters(req, constraints); if (p.error) { - res.status(501).json(this.getBisqMarketErrorResponse(p.error)); + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); return; } @@ -197,7 +197,7 @@ class Routes { const p = this.parseRequestParameters(req, constraints); if (p.error) { - res.status(501).json(this.getBisqMarketErrorResponse(p.error)); + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); return; } @@ -256,7 +256,7 @@ class Routes { const p = this.parseRequestParameters(req, constraints); if (p.error) { - res.status(501).json(this.getBisqMarketErrorResponse(p.error)); + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); return; } @@ -283,7 +283,7 @@ class Routes { const p = this.parseRequestParameters(req, constraints); if (p.error) { - res.status(501).json(this.getBisqMarketErrorResponse(p.error)); + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); return; } @@ -298,7 +298,7 @@ class Routes { public getBisqMarketVolumes(req: Request, res: Response) { const constraints: RequiredSpec = { 'market': { - required: true, + required: false, types: ['@string'] }, 'interval': { @@ -313,15 +313,19 @@ class Routes { required: false, types: ['@number'] }, + 'milliseconds': { + required: false, + types: ['@boolean'] + }, }; const p = this.parseRequestParameters(req, constraints); if (p.error) { - res.status(501).json(this.getBisqMarketErrorResponse(p.error)); + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); return; } - const result = bisqMarket.getVolumes(p.timestamp_from, p.timestamp_to, p.interval, p.market); + const result = bisqMarket.getVolumes(p.market, p.timestamp_from, p.timestamp_to, p.interval, p.milliseconds); if (result) { res.json(result); } else { @@ -355,7 +359,7 @@ class Routes { const p = this.parseRequestParameters(req, constraints); if (p.error) { - res.status(501).json(this.getBisqMarketErrorResponse(p.error)); + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); return; } @@ -377,11 +381,11 @@ class Routes { const p = this.parseRequestParameters(req, constraints); if (p.error) { - res.status(501).json(this.getBisqMarketErrorResponse(p.error)); + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); return; } - const result = bisqMarket.getCurrencies(p.market); + const result = bisqMarket.getTicker(p.market); if (result) { res.json(result); } else { diff --git a/production/mempool-config.bisq.json b/production/mempool-config.bisq.json index aeb1c9033..630790990 100644 --- a/production/mempool-config.bisq.json +++ b/production/mempool-config.bisq.json @@ -14,7 +14,7 @@ "TX_PER_SECOND_SPAN_SECONDS": 150, "ELECTRS_API_URL": "http://[::1]:3000", "BISQ_ENABLED": true, - "BSQ_BLOCKS_DATA_PATH": "/bisq/data", + "BISQ_BLOCKS_DATA_PATH": "/bisq/data", "SSL": false, "SSL_CERT_FILE_PATH": "/etc/letsencrypt/live/mysite/fullchain.pem", "SSL_KEY_FILE_PATH": "/etc/letsencrypt/live/mysite/privkey.pem"