From 1ae34e069c1056f87cf7a7747df5d338c74643ef Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 14 Nov 2023 05:00:05 +0000 Subject: [PATCH 1/4] Fix silently unhandled database exceptions --- backend/src/api/blocks.ts | 9 +++++++-- backend/src/database.ts | 10 ++++++++-- backend/src/index.ts | 20 +++++++++++++++----- backend/src/indexer.ts | 7 ++++++- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 64dc1d5ba..730e603f3 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -761,8 +761,13 @@ class Blocks { this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`); if (!fastForwarded) { - const lastestPriceId = await PricesRepository.$getLatestPriceId(); - this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`); + let lastestPriceId; + try { + lastestPriceId = await PricesRepository.$getLatestPriceId(); + this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`); + } catch (e) { + logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e)); + } if (priceUpdater.historyInserted === true && lastestPriceId !== null) { await blocksRepository.$saveBlockPrices([{ height: blockExtended.height, diff --git a/backend/src/database.ts b/backend/src/database.ts index 0638aa319..21d90261d 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -55,14 +55,20 @@ import { execSync } from 'child_process'; }).then(result => { resolve(result); }).catch(error => { + logger.debug(`database query "${query.slice(0, 100)}" failed!`); reject(error); }).finally(() => { clearTimeout(timer); }); }); } else { - const pool = await this.getPool(); - return pool.query(query, params); + try { + const pool = await this.getPool(); + return pool.query(query, params); + } catch (e) { + logger.debug(`database query "${query.slice(0, 100)}" failed!`); + throw e; + } } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 59c92fbf3..039aad8af 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -92,9 +92,15 @@ class Server { logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`); // Register cleanup listeners for exit events - ['exit', 'SIGHUP', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'unhandledRejection'].forEach(event => { + ['exit', 'SIGHUP', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2'].forEach(event => { process.on(event, () => { this.onExit(event); }); }); + process.on('uncaughtException', (error) => { + this.onUnhandledException('uncaughtException', error); + }); + process.on('unhandledRejection', (reason, promise) => { + this.onUnhandledException('unhandledRejection', reason); + }); if (config.MEMPOOL.BACKEND === 'esplora') { bitcoinApi.startHealthChecks(); @@ -314,14 +320,18 @@ class Server { } } - onExit(exitEvent): void { + onExit(exitEvent, code = 0): void { + logger.debug(`onExit for signal: ${exitEvent}`); if (config.DATABASE.ENABLED) { DB.releasePidLock(); } - process.exit(0); + process.exit(code); + } + + onUnhandledException(type, error): void { + console.error(`${type}:`, error); + this.onExit(type, 1); } } - - ((): Server => new Server())(); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 7ec65d9c9..add0abca3 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -76,7 +76,12 @@ class Indexer { if (task === 'blocksPrices' && !this.tasksRunning.includes(task) && !['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { this.tasksRunning.push(task); - const lastestPriceId = await PricesRepository.$getLatestPriceId(); + let lastestPriceId; + try { + lastestPriceId = await PricesRepository.$getLatestPriceId(); + } catch (e) { + logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e)); + } if (priceUpdater.historyInserted === false || lastestPriceId === null) { logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining); setTimeout(() => { From 08b68ef8ba895cb82ad763bd5ab924991a53469a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 14 Nov 2023 05:33:48 +0000 Subject: [PATCH 2/4] handle exception in db transaction rollback --- backend/src/database.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/src/database.ts b/backend/src/database.ts index 21d90261d..a9893fdac 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -72,6 +72,15 @@ import { execSync } from 'child_process'; } } + private async $rollbackAtomic(connection: PoolConnection): Promise { + try { + await connection.rollback(); + await connection.release(); + } catch (e) { + logger.warn('Failed to rollback incomplete db transaction: ' + (e instanceof Error ? e.message : e)); + } + } + public async $atomicQuery(queries: { query, params }[]): Promise<[T, FieldPacket[]][]> { @@ -90,9 +99,8 @@ import { execSync } from 'child_process'; return results; } catch (e) { - logger.err('Could not complete db transaction, rolling back: ' + (e instanceof Error ? e.message : e)); - connection.rollback(); - connection.release(); + logger.warn('Could not complete db transaction, rolling back: ' + (e instanceof Error ? e.message : e)); + this.$rollbackAtomic(connection); throw e; } finally { connection.release(); From 29cbdf6cd56457893de25ca2a9542aa81937327a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 14 Nov 2023 05:52:27 +0000 Subject: [PATCH 3/4] handle null query in error log --- backend/src/database.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/database.ts b/backend/src/database.ts index a9893fdac..9d21ac28e 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -55,7 +55,7 @@ import { execSync } from 'child_process'; }).then(result => { resolve(result); }).catch(error => { - logger.debug(`database query "${query.slice(0, 100)}" failed!`); + logger.debug(`database query "${query?.slice(0, 100)}" failed!`); reject(error); }).finally(() => { clearTimeout(timer); @@ -66,7 +66,7 @@ import { execSync } from 'child_process'; const pool = await this.getPool(); return pool.query(query, params); } catch (e) { - logger.debug(`database query "${query.slice(0, 100)}" failed!`); + logger.debug(`database query "${query?.slice(0, 100)}" failed!`); throw e; } } From 5c0a59d2f61a5ee64f9dfd7cc15a39186a5812c1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 14 Nov 2023 06:13:06 +0000 Subject: [PATCH 4/4] handle unknown query types in db error log --- backend/src/database.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/database.ts b/backend/src/database.ts index 9d21ac28e..4eba8cb6f 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -55,7 +55,7 @@ import { execSync } from 'child_process'; }).then(result => { resolve(result); }).catch(error => { - logger.debug(`database query "${query?.slice(0, 100)}" failed!`); + logger.debug(`database query "${query?.sql?.slice(0, 160) || (typeof(query) === 'string' || query instanceof String ? query?.slice(0, 160) : 'unknown query')}" failed!`); reject(error); }).finally(() => { clearTimeout(timer); @@ -66,7 +66,7 @@ import { execSync } from 'child_process'; const pool = await this.getPool(); return pool.query(query, params); } catch (e) { - logger.debug(`database query "${query?.slice(0, 100)}" failed!`); + logger.debug(`database query "${query?.sql?.slice(0, 160) || (typeof(query) === 'string' || query instanceof String ? query?.slice(0, 160) : 'unknown query')}" failed!`); throw e; } }