mirror of
https://github.com/mempool/mempool.git
synced 2024-11-20 02:11:49 +01:00
Merge branch 'master' into nymkappa/bugfix/node-null-link
This commit is contained in:
wiz
GitHub
commit
3fd8af0c78
@ -79,8 +79,8 @@
|
||||
},
|
||||
"LND": {
|
||||
"TLS_CERT_PATH": "tls.cert",
|
||||
"MACAROON_PATH": "admin.macaroon",
|
||||
"SOCKET": "localhost:10009"
|
||||
"MACAROON_PATH": "readonly.macaroon",
|
||||
"REST_API_URL": "https://localhost:8080"
|
||||
},
|
||||
"SOCKS5PROXY": {
|
||||
"ENABLED": false,
|
||||
|
||||
892
backend/package-lock.json
generated
892
backend/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -16,7 +16,8 @@
|
||||
"mempool",
|
||||
"blockchain",
|
||||
"explorer",
|
||||
"liquid"
|
||||
"liquid",
|
||||
"lightning"
|
||||
],
|
||||
"main": "index.ts",
|
||||
"scripts": {
|
||||
@ -34,10 +35,8 @@
|
||||
"@types/node": "^16.11.41",
|
||||
"axios": "~0.27.2",
|
||||
"bitcoinjs-lib": "6.0.1",
|
||||
"bolt07": "^1.8.1",
|
||||
"crypto-js": "^4.0.0",
|
||||
"express": "^4.18.0",
|
||||
"lightning": "^5.16.3",
|
||||
"maxmind": "^4.3.6",
|
||||
"mysql2": "2.3.3",
|
||||
"node-worker-threads-pool": "^1.5.1",
|
||||
|
||||
@ -1,71 +1,85 @@
|
||||
export namespace ILightningApi {
|
||||
export interface NetworkInfo {
|
||||
average_channel_size: number;
|
||||
channel_count: number;
|
||||
max_channel_size: number;
|
||||
median_channel_size: number;
|
||||
min_channel_size: number;
|
||||
node_count: number;
|
||||
not_recently_updated_policy_count: number;
|
||||
total_capacity: number;
|
||||
graph_diameter: number;
|
||||
avg_out_degree: number;
|
||||
max_out_degree: number;
|
||||
num_nodes: number;
|
||||
num_channels: number;
|
||||
total_network_capacity: string;
|
||||
avg_channel_size: number;
|
||||
min_channel_size: string;
|
||||
max_channel_size: string;
|
||||
median_channel_size_sat: string;
|
||||
num_zombie_chans: string;
|
||||
}
|
||||
|
||||
export interface NetworkGraph {
|
||||
channels: Channel[];
|
||||
nodes: Node[];
|
||||
edges: Channel[];
|
||||
}
|
||||
|
||||
export interface Channel {
|
||||
id: string;
|
||||
capacity: number;
|
||||
policies: Policy[];
|
||||
transaction_id: string;
|
||||
transaction_vout: number;
|
||||
updated_at?: string;
|
||||
channel_id: string;
|
||||
chan_point: string;
|
||||
last_update: number;
|
||||
node1_pub: string;
|
||||
node2_pub: string;
|
||||
capacity: string;
|
||||
node1_policy: RoutingPolicy | null;
|
||||
node2_policy: RoutingPolicy | null;
|
||||
}
|
||||
|
||||
interface Policy {
|
||||
public_key: string;
|
||||
base_fee_mtokens?: string;
|
||||
cltv_delta?: number;
|
||||
fee_rate?: number;
|
||||
is_disabled?: boolean;
|
||||
max_htlc_mtokens?: string;
|
||||
min_htlc_mtokens?: string;
|
||||
updated_at?: string;
|
||||
export interface RoutingPolicy {
|
||||
time_lock_delta: number;
|
||||
min_htlc: string;
|
||||
fee_base_msat: string;
|
||||
fee_rate_milli_msat: string;
|
||||
disabled: boolean;
|
||||
max_htlc_msat: string;
|
||||
last_update: number;
|
||||
}
|
||||
|
||||
export interface Node {
|
||||
last_update: number;
|
||||
pub_key: string;
|
||||
alias: string;
|
||||
addresses: {
|
||||
network: string;
|
||||
addr: string;
|
||||
}[];
|
||||
color: string;
|
||||
features: Feature[];
|
||||
public_key: string;
|
||||
sockets: string[];
|
||||
updated_at?: string;
|
||||
features: { [key: number]: Feature };
|
||||
}
|
||||
|
||||
export interface Info {
|
||||
chains: string[];
|
||||
color: string;
|
||||
active_channels_count: number;
|
||||
identity_pubkey: string;
|
||||
alias: string;
|
||||
current_block_hash: string;
|
||||
current_block_height: number;
|
||||
features: Feature[];
|
||||
is_synced_to_chain: boolean;
|
||||
is_synced_to_graph: boolean;
|
||||
latest_block_at: string;
|
||||
peers_count: number;
|
||||
pending_channels_count: number;
|
||||
public_key: string;
|
||||
uris: any[];
|
||||
num_pending_channels: number;
|
||||
num_active_channels: number;
|
||||
num_peers: number;
|
||||
block_height: number;
|
||||
block_hash: string;
|
||||
synced_to_chain: boolean;
|
||||
testnet: boolean;
|
||||
uris: string[];
|
||||
best_header_timestamp: string;
|
||||
version: string;
|
||||
num_inactive_channels: number;
|
||||
chains: {
|
||||
chain: string;
|
||||
network: string;
|
||||
}[];
|
||||
color: string;
|
||||
synced_to_graph: boolean;
|
||||
features: { [key: number]: Feature };
|
||||
commit_hash: string;
|
||||
/** Available on LND since v0.15.0-beta */
|
||||
require_htlc_interceptor?: boolean;
|
||||
}
|
||||
|
||||
|
||||
export interface Feature {
|
||||
bit: number;
|
||||
is_known: boolean;
|
||||
name: string;
|
||||
is_required: boolean;
|
||||
type?: string;
|
||||
is_known: boolean;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,44 +1,40 @@
|
||||
import axios, { AxiosRequestConfig } from 'axios';
|
||||
import { Agent } from 'https';
|
||||
import * as fs from 'fs';
|
||||
import { AbstractLightningApi } from '../lightning-api-abstract-factory';
|
||||
import { ILightningApi } from '../lightning-api.interface';
|
||||
import * as fs from 'fs';
|
||||
import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning';
|
||||
import config from '../../../config';
|
||||
import logger from '../../../logger';
|
||||
|
||||
class LndApi implements AbstractLightningApi {
|
||||
private lnd: any;
|
||||
axiosConfig: AxiosRequestConfig = {};
|
||||
|
||||
constructor() {
|
||||
if (!config.LIGHTNING.ENABLED) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64');
|
||||
const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64');
|
||||
|
||||
const { lnd } = authenticatedLndGrpc({
|
||||
cert: tls,
|
||||
macaroon: macaroon,
|
||||
socket: config.LND.SOCKET,
|
||||
});
|
||||
|
||||
this.lnd = lnd;
|
||||
} catch (e) {
|
||||
logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
|
||||
process.exit(1);
|
||||
if (config.LIGHTNING.ENABLED) {
|
||||
this.axiosConfig = {
|
||||
headers: {
|
||||
'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
|
||||
},
|
||||
httpsAgent: new Agent({
|
||||
ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
|
||||
}),
|
||||
timeout: 10000
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
|
||||
return await getNetworkInfo({ lnd: this.lnd });
|
||||
return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
|
||||
.then((response) => response.data);
|
||||
}
|
||||
|
||||
async $getInfo(): Promise<ILightningApi.Info> {
|
||||
// @ts-ignore
|
||||
return await getWalletInfo({ lnd: this.lnd });
|
||||
return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
|
||||
.then((response) => response.data);
|
||||
}
|
||||
|
||||
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
|
||||
return await getNetworkGraph({ lnd: this.lnd });
|
||||
return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
|
||||
.then((response) => response.data);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@ interface IConfig {
|
||||
LND: {
|
||||
TLS_CERT_PATH: string;
|
||||
MACAROON_PATH: string;
|
||||
SOCKET: string;
|
||||
REST_API_URL: string;
|
||||
};
|
||||
ELECTRUM: {
|
||||
HOST: string;
|
||||
@ -182,7 +182,7 @@ const defaults: IConfig = {
|
||||
'LND': {
|
||||
'TLS_CERT_PATH': '',
|
||||
'MACAROON_PATH': '',
|
||||
'SOCKET': 'localhost:10009',
|
||||
'REST_API_URL': 'https://localhost:8080',
|
||||
},
|
||||
'SOCKS5PROXY': {
|
||||
'ENABLED': false,
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import { chanNumber } from 'bolt07';
|
||||
import DB from '../../database';
|
||||
import logger from '../../logger';
|
||||
import channelsApi from '../../api/explorer/channels.api';
|
||||
@ -39,9 +38,9 @@ class NodeSyncService {
|
||||
}
|
||||
|
||||
const graphChannelsIds: string[] = [];
|
||||
for (const channel of networkGraph.channels) {
|
||||
for (const channel of networkGraph.edges) {
|
||||
await this.$saveChannel(channel);
|
||||
graphChannelsIds.push(channel.id);
|
||||
graphChannelsIds.push(channel.channel_id);
|
||||
}
|
||||
await this.$setChannelsInactive(graphChannelsIds);
|
||||
|
||||
@ -56,7 +55,7 @@ class NodeSyncService {
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
|
||||
logger.err('$runUpdater() error: ' + (e instanceof Error ? e.message : e));
|
||||
}
|
||||
}
|
||||
|
||||
@ -107,8 +106,7 @@ class NodeSyncService {
|
||||
logger.info(`Running inactive channels scan...`);
|
||||
|
||||
try {
|
||||
// @ts-ignore
|
||||
const [channels]: [ILightningApi.Channel[]] = await DB.query(`
|
||||
const [channels]: [{ id: string }[]] = await <any>DB.query(`
|
||||
SELECT channels.id
|
||||
FROM channels
|
||||
WHERE channels.status = 1
|
||||
@ -266,7 +264,10 @@ class NodeSyncService {
|
||||
}
|
||||
|
||||
private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
|
||||
const fromChannel = chanNumber({ channel: channel.id }).number;
|
||||
const [ txid, vout ] = channel.chan_point.split(':');
|
||||
|
||||
const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
|
||||
const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
|
||||
|
||||
try {
|
||||
const query = `INSERT INTO channels
|
||||
@ -319,55 +320,55 @@ class NodeSyncService {
|
||||
;`;
|
||||
|
||||
await DB.query(query, [
|
||||
fromChannel,
|
||||
channel.id,
|
||||
channel.channel_id,
|
||||
this.toShortId(channel.channel_id),
|
||||
channel.capacity,
|
||||
channel.transaction_id,
|
||||
channel.transaction_vout,
|
||||
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
|
||||
channel.policies[0].public_key,
|
||||
channel.policies[0].base_fee_mtokens,
|
||||
channel.policies[0].cltv_delta,
|
||||
channel.policies[0].fee_rate,
|
||||
channel.policies[0].is_disabled,
|
||||
channel.policies[0].max_htlc_mtokens,
|
||||
channel.policies[0].min_htlc_mtokens,
|
||||
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
|
||||
channel.policies[1].public_key,
|
||||
channel.policies[1].base_fee_mtokens,
|
||||
channel.policies[1].cltv_delta,
|
||||
channel.policies[1].fee_rate,
|
||||
channel.policies[1].is_disabled,
|
||||
channel.policies[1].max_htlc_mtokens,
|
||||
channel.policies[1].min_htlc_mtokens,
|
||||
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
|
||||
txid,
|
||||
vout,
|
||||
this.utcDateToMysql(channel.last_update),
|
||||
channel.node1_pub,
|
||||
policy1.fee_base_msat,
|
||||
policy1.time_lock_delta,
|
||||
policy1.fee_rate_milli_msat,
|
||||
policy1.disabled,
|
||||
policy1.max_htlc_msat,
|
||||
policy1.min_htlc,
|
||||
this.utcDateToMysql(policy1.last_update),
|
||||
channel.node2_pub,
|
||||
policy2.fee_base_msat,
|
||||
policy2.time_lock_delta,
|
||||
policy2.fee_rate_milli_msat,
|
||||
policy2.disabled,
|
||||
policy2.max_htlc_msat,
|
||||
policy2.min_htlc,
|
||||
this.utcDateToMysql(policy2.last_update),
|
||||
channel.capacity,
|
||||
channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
|
||||
channel.policies[0].public_key,
|
||||
channel.policies[0].base_fee_mtokens,
|
||||
channel.policies[0].cltv_delta,
|
||||
channel.policies[0].fee_rate,
|
||||
channel.policies[0].is_disabled,
|
||||
channel.policies[0].max_htlc_mtokens,
|
||||
channel.policies[0].min_htlc_mtokens,
|
||||
channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
|
||||
channel.policies[1].public_key,
|
||||
channel.policies[1].base_fee_mtokens,
|
||||
channel.policies[1].cltv_delta,
|
||||
channel.policies[1].fee_rate,
|
||||
channel.policies[1].is_disabled,
|
||||
channel.policies[1].max_htlc_mtokens,
|
||||
channel.policies[1].min_htlc_mtokens,
|
||||
channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
|
||||
this.utcDateToMysql(channel.last_update),
|
||||
channel.node1_pub,
|
||||
policy1.fee_base_msat,
|
||||
policy1.time_lock_delta,
|
||||
policy1.fee_rate_milli_msat,
|
||||
policy1.disabled,
|
||||
policy1.max_htlc_msat,
|
||||
policy1.min_htlc,
|
||||
this.utcDateToMysql(policy1.last_update),
|
||||
channel.node2_pub,
|
||||
policy2.fee_base_msat,
|
||||
policy2.time_lock_delta,
|
||||
policy2.fee_rate_milli_msat,
|
||||
policy2.disabled,
|
||||
policy2.max_htlc_msat,
|
||||
policy2.min_htlc,
|
||||
this.utcDateToMysql(policy2.last_update)
|
||||
]);
|
||||
} catch (e) {
|
||||
logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
|
||||
}
|
||||
}
|
||||
|
||||
private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> {
|
||||
private async $updateChannelStatus(channelId: string, status: number): Promise<void> {
|
||||
try {
|
||||
await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]);
|
||||
await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelId]);
|
||||
} catch (e) {
|
||||
logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
|
||||
}
|
||||
@ -390,8 +391,8 @@ class NodeSyncService {
|
||||
|
||||
private async $saveNode(node: ILightningApi.Node): Promise<void> {
|
||||
try {
|
||||
const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00';
|
||||
const sockets = node.sockets.join(',');
|
||||
const updatedAt = this.utcDateToMysql(node.last_update);
|
||||
const sockets = node.addresses.map(a => a.addr).join(',');
|
||||
const query = `INSERT INTO nodes(
|
||||
public_key,
|
||||
first_seen,
|
||||
@ -403,7 +404,7 @@ class NodeSyncService {
|
||||
VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
|
||||
|
||||
await DB.query(query, [
|
||||
node.public_key,
|
||||
node.pub_key,
|
||||
updatedAt,
|
||||
node.alias,
|
||||
node.color,
|
||||
@ -418,8 +419,18 @@ class NodeSyncService {
|
||||
}
|
||||
}
|
||||
|
||||
private utcDateToMysql(dateString: string): string {
|
||||
const d = new Date(Date.parse(dateString));
|
||||
/** Decodes a channel id returned by lnd as uint64 to a short channel id */
|
||||
private toShortId(id: string): string {
|
||||
const n = BigInt(id);
|
||||
return [
|
||||
n >> 40n, // nth block
|
||||
(n >> 16n) & 0xffffffn, // nth tx of the block
|
||||
n & 0xffffn // nth output of the tx
|
||||
].join('x');
|
||||
}
|
||||
|
||||
private utcDateToMysql(date?: number): string {
|
||||
const d = new Date((date || 0) * 1000);
|
||||
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
|
||||
}
|
||||
}
|
||||
|
||||
@ -52,7 +52,7 @@ class LightningStatsUpdater {
|
||||
|
||||
private async $lightningIsSynced(): Promise<boolean> {
|
||||
const nodeInfo = await lightningApi.$getInfo();
|
||||
return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph;
|
||||
return nodeInfo.synced_to_chain && nodeInfo.synced_to_graph;
|
||||
}
|
||||
|
||||
private async $runTasks(): Promise<void> {
|
||||
@ -66,13 +66,13 @@ class LightningStatsUpdater {
|
||||
|
||||
private async $logLightningStatsDaily() {
|
||||
try {
|
||||
logger.info(`Running lightning daily stats log...`);
|
||||
logger.info(`Running lightning daily stats log...`);
|
||||
|
||||
const networkGraph = await lightningApi.$getNetworkGraph();
|
||||
let total_capacity = 0;
|
||||
for (const channel of networkGraph.channels) {
|
||||
for (const channel of networkGraph.edges) {
|
||||
if (channel.capacity) {
|
||||
total_capacity += channel.capacity;
|
||||
total_capacity += parseInt(channel.capacity);
|
||||
}
|
||||
}
|
||||
|
||||
@ -80,20 +80,17 @@ class LightningStatsUpdater {
|
||||
let torNodes = 0;
|
||||
let unannouncedNodes = 0;
|
||||
for (const node of networkGraph.nodes) {
|
||||
let isUnnanounced = true;
|
||||
for (const socket of node.sockets) {
|
||||
const hasOnion = socket.indexOf('.onion') !== -1;
|
||||
for (const socket of node.addresses) {
|
||||
const hasOnion = socket.addr.indexOf('.onion') !== -1;
|
||||
if (hasOnion) {
|
||||
torNodes++;
|
||||
isUnnanounced = false;
|
||||
}
|
||||
const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
|
||||
const hasClearnet = [4, 6].includes(net.isIP(socket.addr.split(':')[0]));
|
||||
if (hasClearnet) {
|
||||
clearnetNodes++;
|
||||
isUnnanounced = false;
|
||||
}
|
||||
}
|
||||
if (isUnnanounced) {
|
||||
if (node.addresses.length === 0) {
|
||||
unannouncedNodes++;
|
||||
}
|
||||
}
|
||||
@ -118,7 +115,7 @@ class LightningStatsUpdater {
|
||||
VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
||||
|
||||
await DB.query(query, [
|
||||
networkGraph.channels.length,
|
||||
networkGraph.edges.length,
|
||||
networkGraph.nodes.length,
|
||||
total_capacity,
|
||||
torNodes,
|
||||
@ -292,7 +289,7 @@ class LightningStatsUpdater {
|
||||
|
||||
for (const node of nodes) {
|
||||
const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]);
|
||||
|
||||
|
||||
const date: Date = new Date(this.hardCodedStartTime);
|
||||
const currentDate = new Date();
|
||||
this.setDateMidnight(currentDate);
|
||||
@ -322,7 +319,7 @@ class LightningStatsUpdater {
|
||||
|
||||
lastTotalCapacity = totalCapacity;
|
||||
lastChannelsCount = channelsCount;
|
||||
|
||||
|
||||
const query = `INSERT INTO node_stats(
|
||||
public_key,
|
||||
added,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user