Merge pull request #799 from mempool/simon/liquid-lbtc-widget

Liquid L-BTC widgets (Backend)
This commit is contained in:
wiz 2021-09-24 23:54:33 +09:00 committed by GitHub
commit 800625d80e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 243 additions and 95 deletions

View File

@ -12,7 +12,8 @@
"BLOCK_WEIGHT_UNITS": 4000000,
"INITIAL_BLOCKS_AMOUNT": 8,
"MEMPOOL_BLOCKS_AMOUNT": 8,
"PRICE_FEED_UPDATE_INTERVAL": 3600
"PRICE_FEED_UPDATE_INTERVAL": 3600,
"USE_SECOND_NODE_FOR_MINFEE": false
},
"CORE_RPC": {
"HOST": "127.0.0.1",
@ -28,8 +29,7 @@
"ESPLORA": {
"REST_API_URL": "http://127.0.0.1:3000"
},
"CORE_RPC_MINFEE": {
"ENABLED": false,
"SECOND_CORE_RPC": {
"HOST": "127.0.0.1",
"PORT": 8332,
"USERNAME": "mempool",

View File

@ -12,3 +12,10 @@ export interface AbstractBitcoinApi {
$getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$getAddressPrefix(prefix: string): string[];
}
export interface BitcoinRpcCredentials {
host: string;
port: number;
user: string;
pass: string;
timeout: number;
}

View File

@ -3,16 +3,17 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import EsploraApi from './esplora-api';
import BitcoinApi from './bitcoin-api';
import ElectrumApi from './electrum-api';
import bitcoinClient from './bitcoin-client';
function bitcoinApiFactory(): AbstractBitcoinApi {
switch (config.MEMPOOL.BACKEND) {
case 'esplora':
return new EsploraApi();
case 'electrum':
return new ElectrumApi();
return new ElectrumApi(bitcoinClient);
case 'none':
default:
return new BitcoinApi();
return new BitcoinApi(bitcoinClient);
}
}

View File

@ -72,7 +72,7 @@ export namespace IBitcoinApi {
time: number; // (numeric) Same as blocktime
}
interface Vin {
export interface Vin {
txid?: string; // (string) The transaction id
vout?: number; // (string)
scriptSig?: { // (json object) The script
@ -82,18 +82,22 @@ export namespace IBitcoinApi {
sequence: number; // (numeric) The script sequence number
txinwitness?: string[]; // (string) hex-encoded witness data
coinbase?: string;
is_pegin?: boolean; // (boolean) Elements peg-in
}
interface Vout {
export interface Vout {
value: number; // (numeric) The value in BTC
n: number; // (numeric) index
asset?: string; // (string) Elements asset id
scriptPubKey: { // (json object)
asm: string; // (string) the asm
hex: string; // (string) the hex
reqSigs?: number; // (numeric) The required sigs
type: string; // (string) The type, eg 'pubkeyhash'
addresses?: string[]; // (string) bitcoin addresses
address?: string; // (string) bitcoin address
addresses?: string[]; // (string) bitcoin addresses
pegout_chain?: string; // (string) Elements peg-out chain
pegout_addresses?: string[]; // (string) Elements peg-out addresses
};
}

View File

@ -1,5 +1,3 @@
import config from '../../config';
import * as bitcoin from '@mempool/bitcoin';
import * as bitcoinjs from 'bitcoinjs-lib';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IBitcoinApi } from './bitcoin-api.interface';
@ -10,16 +8,10 @@ import { TransactionExtended } from '../../mempool.interfaces';
class BitcoinApi implements AbstractBitcoinApi {
private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
private bitcoindClient: any;
protected bitcoindClient: any;
constructor() {
this.bitcoindClient = new bitcoin.Client({
host: config.CORE_RPC.HOST,
port: config.CORE_RPC.PORT,
user: config.CORE_RPC.USERNAME,
pass: config.CORE_RPC.PASSWORD,
timeout: 60000,
});
constructor(bitcoinClient: any) {
this.bitcoindClient = bitcoinClient;
}
$getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {

View File

@ -1,53 +0,0 @@
import config from '../../config';
import * as bitcoin from '@mempool/bitcoin';
import { IBitcoinApi } from './bitcoin-api.interface';
class BitcoinBaseApi {
bitcoindClient: any;
bitcoindClientMempoolInfo: any;
constructor() {
this.bitcoindClient = new bitcoin.Client({
host: config.CORE_RPC.HOST,
port: config.CORE_RPC.PORT,
user: config.CORE_RPC.USERNAME,
pass: config.CORE_RPC.PASSWORD,
timeout: 60000,
});
if (config.CORE_RPC_MINFEE.ENABLED) {
this.bitcoindClientMempoolInfo = new bitcoin.Client({
host: config.CORE_RPC_MINFEE.HOST,
port: config.CORE_RPC_MINFEE.PORT,
user: config.CORE_RPC_MINFEE.USERNAME,
pass: config.CORE_RPC_MINFEE.PASSWORD,
timeout: 60000,
});
}
}
$getMempoolInfo(): Promise<IBitcoinApi.MempoolInfo> {
if (config.CORE_RPC_MINFEE.ENABLED) {
return Promise.all([
this.bitcoindClient.getMempoolInfo(),
this.bitcoindClientMempoolInfo.getMempoolInfo()
]).then(([mempoolInfo, secondMempoolInfo]) => {
mempoolInfo.maxmempool = secondMempoolInfo.maxmempool;
mempoolInfo.mempoolminfee = secondMempoolInfo.mempoolminfee;
mempoolInfo.minrelaytxfee = secondMempoolInfo.minrelaytxfee;
return mempoolInfo;
});
}
return this.bitcoindClient.getMempoolInfo();
}
$getBlockchainInfo(): Promise<IBitcoinApi.BlockchainInfo> {
return this.bitcoindClient.getBlockchainInfo();
}
$validateAddress(address: string): Promise<IBitcoinApi.AddressInformation> {
return this.bitcoindClient.validateAddress(address);
}
}
export default new BitcoinBaseApi();

View File

@ -0,0 +1,13 @@
import config from '../../config';
import * as bitcoin from '@mempool/bitcoin';
import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
const nodeRpcCredentials: BitcoinRpcCredentials = {
host: config.CORE_RPC.HOST,
port: config.CORE_RPC.PORT,
user: config.CORE_RPC.USERNAME,
pass: config.CORE_RPC.PASSWORD,
timeout: 60000,
};
export default new bitcoin.Client(nodeRpcCredentials);

View File

@ -0,0 +1,13 @@
import config from '../../config';
import * as bitcoin from '@mempool/bitcoin';
import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
const nodeRpcCredentials: BitcoinRpcCredentials = {
host: config.SECOND_CORE_RPC.HOST,
port: config.SECOND_CORE_RPC.PORT,
user: config.SECOND_CORE_RPC.USERNAME,
pass: config.SECOND_CORE_RPC.PASSWORD,
timeout: 60000,
};
export default new bitcoin.Client(nodeRpcCredentials);

View File

@ -1,23 +1,20 @@
import config from '../../config';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IBitcoinApi } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
import { IElectrumApi } from './electrum-api.interface';
import BitcoinApi from './bitcoin-api';
import mempool from '../mempool';
import logger from '../../logger';
import * as ElectrumClient from '@mempool/electrum-client';
import * as sha256 from 'crypto-js/sha256';
import * as hexEnc from 'crypto-js/enc-hex';
import loadingIndicators from '../loading-indicators';
import memoryCache from '../memory-cache';
import bitcoinBaseApi from './bitcoin-base.api';
class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
private electrumClient: any;
constructor() {
super();
constructor(bitcoinClient: any) {
super(bitcoinClient);
const electrumConfig = { client: 'mempool-v2', version: '1.4' };
const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };
@ -45,7 +42,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
}
async $getAddress(address: string): Promise<IEsploraApi.Address> {
const addressInfo = await bitcoinBaseApi.$validateAddress(address);
const addressInfo = await this.bitcoindClient.validateAddress(address);
if (!addressInfo || !addressInfo.isvalid) {
return ({
'address': address,
@ -99,7 +96,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
}
async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
const addressInfo = await bitcoinBaseApi.$validateAddress(address);
const addressInfo = await this.bitcoindClient.validateAddress(address);
if (!addressInfo || !addressInfo.isvalid) {
return [];
}

View File

@ -6,7 +6,7 @@ import { BlockExtended, TransactionExtended } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
import bitcoinBaseApi from './bitcoin/bitcoin-base.api';
import bitcoinClient from './bitcoin/bitcoin-client';
class Blocks {
private blocks: BlockExtended[] = [];
@ -45,7 +45,7 @@ class Blocks {
}
if (!this.lastDifficultyAdjustmentTime) {
const blockchainInfo = await bitcoinBaseApi.$getBlockchainInfo();
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
if (blockchainInfo.blocks === blockchainInfo.headers) {
const heightDiff = blockHeightTip % 2016;
const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);

View File

@ -1,6 +1,8 @@
import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
import config from '../config';
export class Common {
static nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
static median(numbers: number[]) {
let medianNr = 0;
const numsLen = numbers.length;

View File

@ -0,0 +1,111 @@
import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface';
import bitcoinClient from '../bitcoin/bitcoin-client';
import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
import { Common } from '../common';
import { DB } from '../../database';
import logger from '../../logger';
class ElementsParser {
private isRunning = false;
constructor() { }
public async $parse() {
if (this.isRunning) {
return;
}
try {
this.isRunning = true;
const result = await bitcoinClient.getChainTips();
const tip = result[0].height;
const latestBlock = await this.$getLatestBlockFromDatabase();
for (let height = latestBlock.block + 1; height <= tip; height++) {
const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height);
const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2);
await this.$parseBlock(block);
await this.$saveLatestBlockToDatabase(block.height, block.time, block.hash);
}
this.isRunning = false;
} catch (e) {
this.isRunning = false;
throw new Error(e instanceof Error ? e.message : 'Error');
}
}
public async $getPegDataByMonth(): Promise<any> {
const connection = await DB.pool.getConnection();
const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
const [rows] = await connection.query<any>(query);
connection.release();
return rows;
}
protected async $parseBlock(block: IBitcoinApi.Block) {
for (const tx of block.tx) {
await this.$parseInputs(tx, block);
await this.$parseOutputs(tx, block);
}
}
protected async $parseInputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
for (const [index, input] of tx.vin.entries()) {
if (input.is_pegin) {
await this.$parsePegIn(input, index, tx.txid, block);
}
}
}
protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) {
const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true);
const prevout = bitcoinTx.vout[input.vout || 0];
const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || '';
await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex,
outputAddress, bitcoinTx.txid, prevout.n, 1);
}
protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
for (const output of tx.vout) {
if (output.scriptPubKey.pegout_chain) {
await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
(output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0);
}
if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata'
&& output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) {
await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
(output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1);
}
}
}
protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
const connection = await DB.pool.getConnection();
const query = `INSERT INTO elements_pegs(
block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const params: (string | number)[] = [
height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
];
await connection.query(query, params);
connection.release();
logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
}
protected async $getLatestBlockFromDatabase(): Promise<any> {
const connection = await DB.pool.getConnection();
const query = `SELECT block, datetime, block_hash FROM last_elements_block`;
const [rows] = await connection.query<any>(query);
connection.release();
return rows[0];
}
protected async $saveLatestBlockToDatabase(blockHeight: number, datetime: number, blockHash: string) {
const connection = await DB.pool.getConnection();
const query = `UPDATE last_elements_block SET block = ?, datetime = ?, block_hash = ?`;
await connection.query<any>(query, [blockHeight, datetime, blockHash]);
connection.release();
}
}
export default new ElementsParser();

View File

@ -5,8 +5,9 @@ import logger from '../logger';
import { Common } from './common';
import transactionUtils from './transaction-utils';
import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
import bitcoinBaseApi from './bitcoin/bitcoin-base.api';
import loadingIndicators from './loading-indicators';
import bitcoinClient from './bitcoin/bitcoin-client';
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
class Mempool {
private static WEBSOCKET_REFRESH_RATE_MS = 10000;
@ -61,7 +62,7 @@ class Mempool {
}
public async $updateMemPoolInfo() {
this.mempoolInfo = await bitcoinBaseApi.$getMempoolInfo();
this.mempoolInfo = await this.$getMempoolInfo();
}
public getMempoolInfo(): IBitcoinApi.MempoolInfo {
@ -205,6 +206,21 @@ class Mempool {
}
}
}
private $getMempoolInfo() {
if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
return Promise.all([
bitcoinClient.getMempoolInfo(),
bitcoinSecondClient.getMempoolInfo()
]).then(([mempoolInfo, secondMempoolInfo]) => {
mempoolInfo.maxmempool = secondMempoolInfo.maxmempool;
mempoolInfo.mempoolminfee = secondMempoolInfo.mempoolminfee;
mempoolInfo.minrelaytxfee = secondMempoolInfo.minrelaytxfee;
return mempoolInfo;
});
}
return bitcoinClient.getMempoolInfo();
}
}
export default new Mempool();

View File

@ -14,7 +14,6 @@ import transactionUtils from './transaction-utils';
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
private nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
private extraInitProperties = {};
constructor() { }
@ -308,7 +307,7 @@ class WebsocketHandler {
newTransactions.forEach((tx) => {
if (client['track-asset'] === this.nativeAssetId) {
if (client['track-asset'] === Common.nativeAssetId) {
if (tx.vin.some((vin) => !!vin.is_pegin)) {
foundTransactions.push(tx);
return;
@ -439,7 +438,7 @@ class WebsocketHandler {
const foundTransactions: TransactionExtended[] = [];
transactions.forEach((tx) => {
if (client['track-asset'] === this.nativeAssetId) {
if (client['track-asset'] === Common.nativeAssetId) {
if (tx.vin && tx.vin.some((vin) => !!vin.is_pegin)) {
foundTransactions.push(tx);
return;

View File

@ -15,6 +15,7 @@ interface IConfig {
INITIAL_BLOCKS_AMOUNT: number;
MEMPOOL_BLOCKS_AMOUNT: number;
PRICE_FEED_UPDATE_INTERVAL: number;
USE_SECOND_NODE_FOR_MINFEE: boolean;
};
ESPLORA: {
REST_API_URL: string;
@ -30,8 +31,7 @@ interface IConfig {
USERNAME: string;
PASSWORD: string;
};
CORE_RPC_MINFEE: {
ENABLED: boolean;
SECOND_CORE_RPC: {
HOST: string;
PORT: number;
USERNAME: string;
@ -77,6 +77,7 @@ const defaults: IConfig = {
'INITIAL_BLOCKS_AMOUNT': 8,
'MEMPOOL_BLOCKS_AMOUNT': 8,
'PRICE_FEED_UPDATE_INTERVAL': 3600,
'USE_SECOND_NODE_FOR_MINFEE': false,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',
@ -92,8 +93,7 @@ const defaults: IConfig = {
'USERNAME': 'mempool',
'PASSWORD': 'mempool'
},
'CORE_RPC_MINFEE': {
'ENABLED': false,
'SECOND_CORE_RPC': {
'HOST': '127.0.0.1',
'PORT': 8332,
'USERNAME': 'mempool',
@ -129,7 +129,7 @@ class Config implements IConfig {
ESPLORA: IConfig['ESPLORA'];
ELECTRUM: IConfig['ELECTRUM'];
CORE_RPC: IConfig['CORE_RPC'];
CORE_RPC_MINFEE: IConfig['CORE_RPC_MINFEE'];
SECOND_CORE_RPC: IConfig['SECOND_CORE_RPC'];
DATABASE: IConfig['DATABASE'];
SYSLOG: IConfig['SYSLOG'];
STATISTICS: IConfig['STATISTICS'];
@ -141,7 +141,7 @@ class Config implements IConfig {
this.ESPLORA = configs.ESPLORA;
this.ELECTRUM = configs.ELECTRUM;
this.CORE_RPC = configs.CORE_RPC;
this.CORE_RPC_MINFEE = configs.CORE_RPC_MINFEE;
this.SECOND_CORE_RPC = configs.SECOND_CORE_RPC;
this.DATABASE = configs.DATABASE;
this.SYSLOG = configs.SYSLOG;
this.STATISTICS = configs.STATISTICS;

View File

@ -20,6 +20,7 @@ import logger from './logger';
import backendInfo from './api/backend-info';
import loadingIndicators from './api/loading-indicators';
import mempool from './api/mempool';
import elementsParser from './api/liquid/elements-parser';
class Server {
private wss: WebSocket.Server | undefined;
@ -112,7 +113,7 @@ class Server {
await memPool.$updateMemPoolInfo();
} catch (e) {
const msg = `updateMempoolInfo: ${(e instanceof Error ? e.message : e)}`;
if (config.CORE_RPC_MINFEE.ENABLED) {
if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
logger.warn(msg);
} else {
logger.debug(msg);
@ -141,6 +142,15 @@ class Server {
if (this.wss) {
websocketHandler.setWebsocketServer(this.wss);
}
if (config.MEMPOOL.NETWORK === 'liquid') {
blocks.setNewBlockCallback(async () => {
try {
await elementsParser.$parse();
} catch (e) {
logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e));
}
});
}
websocketHandler.setupConnectionHandling();
statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
@ -254,6 +264,12 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix)
;
}
if (config.MEMPOOL.NETWORK === 'liquid') {
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)
;
}
}
}

View File

@ -17,7 +17,8 @@ import transactionUtils from './api/transaction-utils';
import blocks from './api/blocks';
import loadingIndicators from './api/loading-indicators';
import { Common } from './api/common';
import bitcoinBaseApi from './api/bitcoin/bitcoin-base.api';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import elementsParser from './api/liquid/elements-parser';
class Routes {
constructor() {}
@ -690,7 +691,7 @@ class Routes {
public async validateAddress(req: Request, res: Response) {
try {
const result = await bitcoinBaseApi.$validateAddress(req.params.address);
const result = await bitcoinClient.validateAddress(req.params.address);
res.json(result);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
@ -754,6 +755,15 @@ class Routes {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async $getElementsPegsByMonth(req: Request, res: Response) {
try {
const pegs = await elementsParser.$getPegDataByMonth();
res.json(pegs);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
export default new Routes();

View File

@ -84,3 +84,23 @@ ALTER TABLE `transactions`
ALTER TABLE `statistics`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
CREATE TABLE `last_elements_block` (
`block` int(11) NOT NULL,
`datetime` int(11) NOT NULL,
`block_hash` varchar(65) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
INSERT INTO `last_elements_block` VALUES(0, 0, '');
CREATE TABLE `elements_pegs` (
`block` int(11) NOT NULL,
`datetime` int(11) NOT NULL,
`amount` bigint(20) NOT NULL,
`txid` varchar(65) NOT NULL,
`txindex` int(11) NOT NULL,
`bitcoinaddress` varchar(100) NOT NULL,
`bitcointxid` varchar(65) NOT NULL,
`bitcoinindex` int(11) NOT NULL,
`final_tx` int(11) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;