diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 9950cdcb3..e6d74ddf0 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -84,19 +84,19 @@ class BitcoinApi implements AbstractBitcoinApi { } $getAddressPrefix(prefix: string): string[] { - const found: string[] = []; + const found: { [address: string]: string } = {}; const mp = mempool.getMempool(); for (const tx in mp) { for (const vout of mp[tx].vout) { if (vout.scriptpubkey_address.indexOf(prefix) === 0) { - found.push(vout.scriptpubkey_address); - if (found.length >= 10) { - return found; + found[vout.scriptpubkey_address] = ''; + if (Object.keys(found).length >= 10) { + return Object.keys(found); } } } } - return found; + return Object.keys(found); } $sendRawTransaction(rawTransaction: string): Promise { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index af25b957b..d6d37b5c4 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -218,31 +218,28 @@ class Blocks { if (blockHeight < lastBlockToIndex) { break; } - try { - ++indexedThisRun; - if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) { - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); - const progress = Math.round(totaIndexed / indexingBlockAmount * 100); - const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds); - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`); - } - const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block = await bitcoinApi.$getBlock(blockHash); - const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); - const blockExtended = await this.$getBlockExtended(block, transactions); - await blocksRepository.$saveBlockInDatabase(blockExtended); - } catch (e) { - logger.err(`Something went wrong while indexing blocks.` + e); + ++indexedThisRun; + if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) { + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); + const progress = Math.round(totaIndexed / indexingBlockAmount * 100); + const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds); + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`); } + const blockHash = await bitcoinApi.$getBlockHash(blockHeight); + const block = await bitcoinApi.$getBlock(blockHash); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); + const blockExtended = await this.$getBlockExtended(block, transactions); + await blocksRepository.$saveBlockInDatabase(blockExtended); } currentBlockHeight -= chunkSize; } logger.info('Block indexing completed'); } catch (e) { - logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e); - console.log(e); + logger.err('An error occured in $generateBlockDatabase(). Trying again later. ' + e); + this.blockIndexingStarted = false; + return; } this.blockIndexingCompleted = true; diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index c0970bf24..1d2f47561 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -90,7 +90,7 @@ class Mining { /** * Return the historical hashrates and oldest indexed block timestamp for one or all pools */ - public async $getPoolsHistoricalHashrates(interval: string | null, poolId: number): Promise { + public async $getPoolsHistoricalHashrates(interval: string | null, poolId: number): Promise { return await HashratesRepository.$getPoolsWeeklyHashrate(interval); } @@ -108,106 +108,112 @@ class Mining { if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) { return; } - this.hashrateIndexingStarted = true; - logger.info(`Indexing hashrates`); + try { + this.hashrateIndexingStarted = true; - const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; - const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); - let startedAt = new Date().getTime() / 1000; - const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - const lastMidnight = new Date(); - lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0); - let toTimestamp = Math.round(lastMidnight.getTime() / 1000); - let indexedThisRun = 0; - let totalIndexed = 0; + logger.info(`Indexing hashrates`); - const hashrates: any[] = []; + const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; + const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); + let startedAt = new Date().getTime() / 1000; + const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + const lastMidnight = new Date(); + lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0); + let toTimestamp = Math.round(lastMidnight.getTime() / 1000); + let indexedThisRun = 0; + let totalIndexed = 0; - while (toTimestamp > genesisTimestamp) { - const fromTimestamp = toTimestamp - 86400; - if (indexedTimestamp.includes(fromTimestamp)) { - toTimestamp -= 86400; - ++totalIndexed; - continue; - } + const hashrates: any[] = []; - const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( - null, fromTimestamp, toTimestamp); - if (blockStats.blockCount === 0) { // We are done indexing, no blocks left - break; - } - - let lastBlockHashrate = 0; - lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, - blockStats.lastBlockHeight); - - if (totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate - logger.debug("Indexing weekly hashrates for mining pools"); - let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp); - const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); - pools = pools.map((pool: any) => { - pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; - pool.share = (pool.blockCount / totalBlocks); - return pool; - }); - - for (const pool of pools) { - hashrates.push({ - hashrateTimestamp: fromTimestamp + 1, - avgHashrate: pool['hashrate'], - poolId: pool.poolId, - share: pool['share'], - type: 'weekly', - }); + while (toTimestamp > genesisTimestamp) { + const fromTimestamp = toTimestamp - 86400; + if (indexedTimestamp.includes(fromTimestamp)) { + toTimestamp -= 86400; + ++totalIndexed; + continue; } + + const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( + null, fromTimestamp, toTimestamp); + if (blockStats.blockCount === 0) { // We are done indexing, no blocks left + break; + } + + let lastBlockHashrate = 0; + lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + blockStats.lastBlockHeight); + + if (totalIndexed > 7 && totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate + logger.debug("Indexing weekly hashrates for mining pools"); + let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp); + const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); + pools = pools.map((pool: any) => { + pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; + pool.share = (pool.blockCount / totalBlocks); + return pool; + }); + + for (const pool of pools) { + hashrates.push({ + hashrateTimestamp: fromTimestamp + 1, + avgHashrate: pool['hashrate'], + poolId: pool.poolId, + share: pool['share'], + type: 'weekly', + }); + } + } + + hashrates.push({ + hashrateTimestamp: fromTimestamp, + avgHashrate: lastBlockHashrate, + poolId: null, + share: 1, + type: 'daily', + }); + + if (hashrates.length > 10) { + await HashratesRepository.$saveHashrates(hashrates); + hashrates.length = 0; + } + + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + if (elapsedSeconds > 5) { + const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2); + const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); + const daysLeft = Math.round(totalDayIndexed - totalIndexed); + logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`); + startedAt = new Date().getTime() / 1000; + indexedThisRun = 0; + } + + toTimestamp -= 86400; + ++indexedThisRun; + ++totalIndexed; } - hashrates.push({ - hashrateTimestamp: fromTimestamp, - avgHashrate: lastBlockHashrate, - poolId: null, - share: 1, - type: 'daily', - }); + // Add genesis block manually + if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) { + hashrates.push({ + hashrateTimestamp: genesisTimestamp, + avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1), + poolId: null, + type: 'daily', + }); + } - if (hashrates.length > 10) { + if (hashrates.length > 0) { await HashratesRepository.$saveHashrates(hashrates); - hashrates.length = 0; } + await HashratesRepository.$setLatestRunTimestamp(); + this.hashrateIndexingStarted = false; - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - if (elapsedSeconds > 5) { - const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2); - const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); - const daysLeft = Math.round(totalDayIndexed - totalIndexed); - logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`); - startedAt = new Date().getTime() / 1000; - indexedThisRun = 0; - } - - toTimestamp -= 86400; - ++indexedThisRun; - ++totalIndexed; + logger.info(`Hashrates indexing completed`); + } catch (e) { + this.hashrateIndexingStarted = false; + throw e; } - - // Add genesis block manually - if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) { - hashrates.push({ - hashrateTimestamp: genesisTimestamp, - avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1), - poolId: null, - type: 'daily', - }); - } - - if (hashrates.length > 0) { - await HashratesRepository.$saveHashrates(hashrates); - } - await HashratesRepository.$setLatestRunTimestamp(); - this.hashrateIndexingStarted = false; - - logger.info(`Hashrates indexing completed`); } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 4305d6dac..51cee34c4 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -96,8 +96,8 @@ class Server { await Common.sleep(5000); await databaseMigration.$truncateIndexedData(tables); } - await this.$resetHashratesIndexingState(); await databaseMigration.$initializeOrMigrateDatabase(); + await this.$resetHashratesIndexingState(); await poolsParser.migratePoolsJson(); } catch (e) { throw new Error(e instanceof Error ? e.message : 'Error'); @@ -167,7 +167,7 @@ class Server { } async $resetHashratesIndexingState() { - return await HashratesRepository.$setLatestRunTimestamp(0); + return await HashratesRepository.$setLatestRunTimestamp(0); } async $runIndexingWhenReady() { diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 212e58327..5a33d7e77 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -22,6 +22,7 @@ class HashratesRepository { await connection.query(query); } catch (e: any) { logger.err('$saveHashrateInDatabase() error' + (e instanceof Error ? e.message : e)); + throw e; } connection.release();