Add git hashes to monitoring

backend/src/api/bitcoin/esplora-api.ts

@@ -1,12 +1,12 @@
 import config from '../../config';
-import axios, { AxiosResponse, isAxiosError } from 'axios';
+import axios, { isAxiosError } from 'axios';
 import http from 'http';
 import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import logger from '../../logger';
 import { Common } from '../common';
 import { SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
-
+import os from 'os';
 interface FailoverHost {
   host: string,
   rtts: number[],
@@ -20,6 +20,12 @@ interface FailoverHost {
   preferred?: boolean,
   checked: boolean,
   lastChecked?: number,
+  hashes: {
+    frontend?: string,
+    backend?: string,
+    electrs?: string,
+    lastUpdated: number,
+  }
 }
 
 class FailoverRouter {
@@ -29,14 +35,21 @@ class FailoverRouter {
   maxHeight: number = 0;
   hosts: FailoverHost[];
   multihost: boolean;
-  pollInterval: number = 60000;
+  gitHashInterval: number = 600000; // 10 minutes
+  pollInterval: number = 60000; // 1 minute
   pollTimer: NodeJS.Timeout | null = null;
   pollConnection = axios.create();
+  localHostname: string = 'localhost';
   requestConnection = axios.create({
     httpAgent: new http.Agent({ keepAlive: true })
   });
 
   constructor() {
+    try {
+      this.localHostname = os.hostname();
+    } catch (e) {
+      logger.warn('Failed to set local hostname, using "localhost"');
+    }
     // setup list of hosts
     this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => {
       return {
@@ -45,6 +58,9 @@ class FailoverRouter {
         rtts: [],
         rtt: Infinity,
         failures: 0,
+        hashes: {
+          lastUpdated: 0,
+        },
       };
     });
     this.activeHost = {
@@ -55,6 +71,9 @@ class FailoverRouter {
       socket: !!config.ESPLORA.UNIX_SOCKET_PATH,
       preferred: true,
       checked: false,
+      hashes: {
+        lastUpdated: 0,
+      },
     };
     this.fallbackHost = this.activeHost;
     this.hosts.unshift(this.activeHost);
@@ -106,6 +125,24 @@ class FailoverRouter {
             host.outOfSync = false;
           }
           host.unreachable = false;
+
+          // update esplora git hash using the x-powered-by header from the height check
+          const poweredBy = result.headers['x-powered-by'];
+          if (poweredBy) {
+            const match = poweredBy.match(/([a-fA-F0-9]{5,40})/);
+            if (match && match[1]?.length) {
+              host.hashes.electrs = match[1];
+            }
+          }
+
+          // Check front and backend git hashes less often
+          if (Date.now() - host.hashes.lastUpdated > this.gitHashInterval) {
+            await Promise.all([
+              this.$updateFrontendGitHash(host),
+              this.$updateBackendGitHash(host)
+            ]);
+            host.hashes.lastUpdated = Date.now();
+          }
         } else {
           host.outOfSync = true;
           host.unreachable = true;
@@ -202,6 +239,32 @@ class FailoverRouter {
     }
   }
 
+  // methods for retrieving git hashes by host
+  private async $updateFrontendGitHash(host: FailoverHost): Promise<void> {
+    try {
+      const url = host.socket ? `http://${this.localHostname}/resources/config.js` : `${host.host.slice(0, -4)}/resources/config.js`;
+      const response = await this.pollConnection.get<string>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      const match = response.data.match(/GIT_COMMIT_HASH\s*=\s*['"](.*?)['"]/);
+      if (match && match[1]?.length) {
+        host.hashes.frontend = match[1];
+      }
+    } catch (e) {
+      // failed to get frontend build hash - do nothing
+    }
+  }
+
+  private async $updateBackendGitHash(host: FailoverHost): Promise<void> {
+    try {
+      const url = host.socket ? `http://${this.localHostname}/api/v1/backend-info` : `${host.host}/v1/backend-info`;
+      const response = await this.pollConnection.get<any>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      if (response.data?.gitCommit) {
+        host.hashes.backend = response.data.gitCommit;
+      }
+    } catch (e) {
+      // failed to get backend build hash - do nothing
+    }
+  }
+
   private async $query<T>(method: 'get'| 'post', path, data: any, responseType = 'json', host = this.activeHost, retry: boolean = true): Promise<T> {
     let axiosConfig;
     let url;
@@ -381,6 +444,7 @@ class ElectrsApi implements AbstractBitcoinApi {
         unreachable: !!host.unreachable,
         checked: !!host.checked,
         lastChecked: host.lastChecked || 0,
+        hashes: host.hashes,
       }));
     } else {
       return [];

frontend/src/app/components/server-health/server-health.component.html

@@ -19,6 +19,9 @@
             <th class="rtt only-small">RTT</th>
             <th class="rtt only-large">RTT</th>
             <th class="height">Height</th>
+            <th class="frontend only-large">Front</th>
+            <th class="backend only-large">Back</th>
+            <th class="electrs only-large">Electrs</th>
           </tr>
           <tr *ngFor="let host of hosts; let i = index; trackBy: trackByFn">
             <td class="rank">{{ i + 1 }}</td>
@@ -28,6 +31,15 @@
             <td class="rtt only-small">{{ (host.rtt / 1000) | number : '1.1-1' }} {{ host.rtt == null ? '' : 's'}} {{ !host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅') }}</td>
             <td class="rtt only-large">{{ host.rtt | number : '1.0-0' }} {{ host.rtt == null ? '' : 'ms'}} {{ !host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅') }}</td>
             <td class="height">{{ host.latestHeight }} {{ !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅')) }}</td>
+            <ng-container *ngFor="let type of ['frontend', 'backend', 'electrs']">
+              <td class="{{type}} only-large" [style.background-color]="host.hashes?.[type] ? '#' + host.hashes[type].slice(0, 6) : ''">
+                @if (host.hashes?.[type]) {
+                  <a [style.color]="'white'" href="https://github.com/mempool/{{type === 'electrs' ? 'electrs' : 'mempool'}}/commit/{{ host.hashes[type] }}" target="_blank">{{ host.hashes[type].slice(0, 8) || '?' }}</a>
+                } @else {
+                  <span>?</span>
+                }
+              </td>
+            </ng-container>
           </tr>
         </tbody>
       </table>

frontend/src/app/components/server-health/server-health.component.scss

@@ -9,7 +9,7 @@
   }
 
   .status-panel {
-    max-width: 720px;
+    max-width: 1000px;
     margin: auto;
     padding: 1em;
     background: var(--box-bg);

frontend/src/app/interfaces/websocket.interface.ts

@@ -144,4 +144,9 @@ export interface HealthCheckHost {
   link?: string;
   statusPage?: SafeResourceUrl;
   flag?: string;
+  hashes?: {
+    frontend?: string;
+    backend?: string;
+    electrs?: string;
+  }
 }