mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2025-02-22 14:22:44 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2074 from mempool/simon/maxmind

Browse source 
Use maxmind to store node locations
This commit is contained in:
wiz 2022-07-11 18:18:00 +02:00 • committed by GitHub
commit 929a4b955c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 171 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
backend/mempool-config.sample.json
View file

@ -63,6 +63,11 @@
"ENABLED": true,
"TX_PER_SECOND_SAMPLE_PERIOD": 150
},
"MAXMIND": {
"ENABLED": false,
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
},
"BISQ": {
"ENABLED": false,
"DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"

50
backend/package-lock.json generated
View file

@ -17,6 +17,7 @@
"crypto-js": "^4.0.0",
"express": "^4.18.0",
"lightning": "^5.16.3",
"maxmind": "^4.3.6",
"mysql2": "2.3.3",
"node-worker-threads-pool": "^1.5.1",
"socks-proxy-agent": "~7.0.0",
@ -2222,6 +2223,19 @@
"node": ">=10"
}
},
"node_modules/maxmind": {
"version": "4.3.6",
"resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz",
"integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==",
"dependencies": {
"mmdb-lib": "2.0.2",
"tiny-lru": "8.0.2"
},
"engines": {
"node": ">=10",
"npm": ">=6"
}
},
"node_modules/md5.js": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@ -2317,6 +2331,15 @@
"node": "*"
}
},
"node_modules/mmdb-lib": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz",
"integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA==",
"engines": {
"node": ">=10",
"npm": ">=6"
}
},
"node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@ -3027,6 +3050,14 @@
"integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
"dev": true
},
"node_modules/tiny-lru": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz",
"integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==",
"engines": {
"node": ">=6"
}
},
"node_modules/tiny-secp256k1": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz",
@ -4971,6 +5002,15 @@
"yallist": "^4.0.0"
}
},
"maxmind": {
"version": "4.3.6",
"resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz",
"integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==",
"requires": {
"mmdb-lib": "2.0.2",
"tiny-lru": "8.0.2"
}
},
"md5.js": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@ -5039,6 +5079,11 @@
"brace-expansion": "^1.1.7"
}
},
"mmdb-lib": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz",
"integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA=="
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@ -5549,6 +5594,11 @@
"integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
"dev": true
},
"tiny-lru": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz",
"integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg=="
},
"tiny-secp256k1": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz",

1
backend/package.json
View file

@ -38,6 +38,7 @@
"crypto-js": "^4.0.0",
"express": "^4.18.0",
"lightning": "^5.16.3",
"maxmind": "^4.3.6",
"mysql2": "2.3.3",
"node-worker-threads-pool": "^1.5.1",
"socks-proxy-agent": "~7.0.0",

23
backend/src/api/database-migration.ts
View file

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 28;
private static currentVersion = 29;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -280,6 +280,17 @@ class DatabaseMigration {
await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
}
if (databaseSchemaVersion < 29 && isBitcoin === true) {
await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names'));
await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
}
} catch (e) {
throw e;
}
@ -693,6 +704,16 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateGeoNamesTableQuery(): string {
return `CREATE TABLE geo_names (
id int(11) unsigned NOT NULL,
type enum('city','country','division','continent') NOT NULL,
names text DEFAULT NULL,
UNIQUE KEY id (id,type),
KEY id_2 (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];

11
backend/src/api/explorer/nodes.api.ts
View file

@ -13,6 +13,17 @@ class NodesApi {
}
}
public async $getAllNodes(): Promise<any> {
try {
const query = `SELECT * FROM nodes`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getAllNodes error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getNodeStats(public_key: string): Promise<any> {
try {
const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;

14
backend/src/config.ts
View file

@ -98,6 +98,11 @@ interface IConfig {
BISQ_URL: string;
BISQ_ONION: string;
};
MAXMIND: {
ENABLED: boolean;
GEOLITE2_CITY: string;
GEOLITE2_ASN: string;
},
}
const defaults: IConfig = {
@ -197,7 +202,12 @@ const defaults: IConfig = {
'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
'BISQ_URL': 'https://bisq.markets/api',
'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
}
},
"MAXMIND": {
'ENABLED': false,
"GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
"GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
},
};
class Config implements IConfig {
@ -215,6 +225,7 @@ class Config implements IConfig {
SOCKS5PROXY: IConfig['SOCKS5PROXY'];
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
MAXMIND: IConfig['MAXMIND'];
constructor() {
const configs = this.merge(configFile, defaults);
@ -232,6 +243,7 @@ class Config implements IConfig {
this.SOCKS5PROXY = configs.SOCKS5PROXY;
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
this.MAXMIND = configs.MAXMIND;
}
merge = (...objects: object[]): IConfig => {

5
backend/src/tasks/lightning/node-sync.service.ts
View file

@ -8,6 +8,7 @@ import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import lightningApi from '../../api/lightning/lightning-api-factory';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
import { $lookupNodeLocation } from './sync-tasks/node-locations';
class NodeSyncService {
constructor() {}
@ -33,6 +34,10 @@ class NodeSyncService {
}
logger.info(`Nodes updated.`);
if (config.MAXMIND.ENABLED) {
await $lookupNodeLocation();
}
await this.$setChannelsInactive();
for (const channel of networkGraph.channels) {

2
backend/src/tasks/lightning/stats-updater.service.ts
View file

@ -206,7 +206,7 @@ class LightningStatsUpdater {
torNodes++;
isUnnanounced = false;
}
const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':'))));
if (hasClearnet) {
clearnetNodes++;
isUnnanounced = false;

63
backend/src/tasks/lightning/sync-tasks/node-locations.ts Normal file
View file

@ -0,0 +1,63 @@
import * as net from 'net';
import maxmind, { CityResponse, AsnResponse } from 'maxmind';
import nodesApi from '../../../api/explorer/nodes.api';
import config from '../../../config';
import DB from '../../../database';
import logger from '../../../logger';
export async function $lookupNodeLocation(): Promise<void> {
logger.info(`Running node location updater using Maxmind...`);
try {
const nodes = await nodesApi.$getAllNodes();
const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
for (const node of nodes) {
const sockets: string[] = node.sockets.split(',');
for (const socket of sockets) {
const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
const hasClearnet = [4, 6].includes(net.isIP(ip));
if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
const city = lookupCity.get(ip);
const asn = lookupAsn.get(ip);
if (city && asn) {
const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`;
const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key];
await DB.query(query, params);
// Store Continent
if (city.continent?.geoname_id) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`,
[city.continent?.geoname_id, JSON.stringify(city.continent?.names)]);
}
// Store Country
if (city.country?.geoname_id) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`,
[city.country?.geoname_id, JSON.stringify(city.country?.names)]);
}
// Store Division
if (city.subdivisions && city.subdivisions[0]) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`,
[city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]);
}
// Store City
if (city.city?.geoname_id) {
await DB.query(
`INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`,
[city.city?.geoname_id, JSON.stringify(city.city?.names)]);
}
}
}
}
}
logger.info(`Node location data updated.`);
} catch (e) {
logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
}
}