mirrors/mempool
1
0
Fork 0
mirror of https://github.com/mempool/mempool.git synced 2024-11-20 10:21:52 +01:00
Code Issues Projects Releases Wiki Activity

Split network daily hashrate indexing and weekly pool hashrate indexing

Browse Source
This commit is contained in:
nymkappa 2022-03-06 12:32:16 +01:00
parent 3e50e4541b
commit 7314582dd1
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04
4 changed files with 139 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/api/database-migration.ts
View File

@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 8;
private static currentVersion = 9;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@ -133,6 +133,10 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
if (databaseSchemaVersion < 9) {
await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)')
}
connection.release();
} catch (e) {
connection.release();
@ -276,6 +280,10 @@ class DatabaseMigration {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
}
if (version < 9) {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
}
return queries;
}

145
backend/src/api/mining.ts
View File

@ -8,6 +8,7 @@ import blocks from './blocks';
class Mining {
hashrateIndexingStarted = false;
weeklyHashrateIndexingStarted = false;
constructor() {
}
@ -95,31 +96,112 @@ class Mining {
}
/**
* Generate daily hashrate data
* Generate weekly mining pool hashrate history
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// We only run this once a day
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp();
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 86400) {
public async $generatePoolHashrateHistory(): Promise<void> {
if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) {
return;
}
// We only run this once a week
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing');
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 604800) {
return;
}
try {
this.weeklyHashrateIndexingStarted = true;
logger.info(`Indexing mining pools weekly hashrates`);
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
const hashrates: any[] = [];
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastWeekMidnight = this.getDateMidnight(new Date());
lastWeekMidnight.setDate(lastWeekMidnight.getDate() - 7);
let toTimestamp = Math.round(lastWeekMidnight.getTime() / 1000);
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 604800;
// Skip already indexed weeks
if (indexedTimestamp.includes(toTimestamp + 1)) {
toTimestamp -= 604800;
continue;
}
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp, toTimestamp);
if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
break;
}
const fromTxt = new Date(1000 * fromTimestamp);
const toTxt = new Date(1000 * toTimestamp);
logger.debug(`Indexing pools hashrate between ${fromTxt.toUTCString()} and ${toTxt.toUTCString()}`)
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp, toTimestamp);
const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
pools = pools.map((pool: any) => {
pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
pool.share = (pool.blockCount / totalBlocks);
return pool;
});
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: toTimestamp + 1,
avgHashrate: pool['hashrate'],
poolId: pool.poolId,
share: pool['share'],
type: 'weekly',
});
}
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
toTimestamp -= 604800;
}
this.weeklyHashrateIndexingStarted = false;
await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing');
logger.info(`Weekly pools hashrate indexing completed`);
} catch (e) {
this.weeklyHashrateIndexingStarted = false;
throw e;
}
}
/**
* Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
return;
}
// We only run this once a day
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_hashrates_indexing');
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 86400) {
return;
}
try {
this.hashrateIndexingStarted = true;
logger.info(`Indexing hashrates`);
logger.info(`Indexing network daily hashrate`);
const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
let startedAt = new Date().getTime() / 1000;
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastMidnight = new Date();
lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0);
const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
let indexedThisRun = 0;
let totalIndexed = 0;
@ -128,6 +210,8 @@ class Mining {
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 86400;
// Skip already indexed weeks
if (indexedTimestamp.includes(fromTimestamp)) {
toTimestamp -= 86400;
++totalIndexed;
@ -140,31 +224,9 @@ class Mining {
break;
}
let lastBlockHashrate = 0;
lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
if (totalIndexed > 7 && totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate
logger.debug("Indexing weekly hashrates for mining pools");
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp);
const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
pools = pools.map((pool: any) => {
pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
pool.share = (pool.blockCount / totalBlocks);
return pool;
});
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: fromTimestamp + 1,
avgHashrate: pool['hashrate'],
poolId: pool.poolId,
share: pool['share'],
type: 'weekly',
});
}
}
hashrates.push({
hashrateTimestamp: fromTimestamp,
avgHashrate: lastBlockHashrate,
@ -203,18 +265,25 @@ class Mining {
});
}
if (hashrates.length > 0) {
await HashratesRepository.$saveHashrates(hashrates);
}
await HashratesRepository.$setLatestRunTimestamp();
this.hashrateIndexingStarted = false;
await HashratesRepository.$saveHashrates(hashrates);
logger.info(`Hashrates indexing completed`);
await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing');
this.hashrateIndexingStarted = false;
logger.info(`Daily network hashrate indexing completed`);
} catch (e) {
this.hashrateIndexingStarted = false;
throw e;
}
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
return date;
}
}
export default new Mining();

4
backend/src/index.ts
View File

@ -167,7 +167,8 @@ class Server {
}
async $resetHashratesIndexingState() {
return await HashratesRepository.$setLatestRunTimestamp(0);
await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRunTimestamp('last_week_hashrates_indexing', 0);
}
async $runIndexingWhenReady() {
@ -178,6 +179,7 @@ class Server {
try {
await blocks.$generateBlockDatabase();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
} catch (e) {
logger.err(`Unable to run indexing right now, trying again later. ` + e);
}

26
backend/src/repositories/HashratesRepository.ts
View File

@ -8,6 +8,10 @@ class HashratesRepository {
* Save indexed block data in the database
*/
public async $saveHashrates(hashrates: any) {
if (hashrates.length === 0) {
return;
}
let query = `INSERT INTO
hashrates(hashrate_timestamp, avg_hashrate, pool_id, share, type) VALUES`;
@ -53,6 +57,16 @@ class HashratesRepository {
return rows;
}
public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
const connection = await DB.pool.getConnection();
let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`;
const [rows]: any[] = await connection.query(query);
connection.release();
return rows.map(row => row.timestamp);
}
/**
* Returns the current biggest pool hashrate history
*/
@ -83,18 +97,18 @@ class HashratesRepository {
return rows;
}
public async $setLatestRunTimestamp(val: any = null) {
public async $setLatestRunTimestamp(key: string, val: any = null) {
const connection = await DB.pool.getConnection();
const query = `UPDATE state SET number = ? WHERE name = 'last_hashrates_indexing'`;
const query = `UPDATE state SET number = ? WHERE name = ?`;
await connection.query<any>(query, (val === null) ? [Math.round(new Date().getTime() / 1000)] : [val]);
await connection.query<any>(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]);
connection.release();
}
public async $getLatestRunTimestamp(): Promise<number> {
public async $getLatestRunTimestamp(key: string): Promise<number> {
const connection = await DB.pool.getConnection();
const query = `SELECT number FROM state WHERE name = 'last_hashrates_indexing'`;
const [rows] = await connection.query<any>(query);
const query = `SELECT number FROM state WHERE name = ?`;
const [rows] = await connection.query<any>(query, [key]);
connection.release();
return rows[0]['number'];
}