Merge pull request #2096 from erikarvstedt/backend-packaging

Simplify packaging for backend

backend/README.md

@@ -110,6 +110,11 @@ Run the Mempool backend:
 
 ```
 npm run start
+
+```
+You can also set env var `MEMPOOL_CONFIG_FILE` to specify a custom config file location:
+```
+MEMPOOL_CONFIG_FILE=/path/to/mempool-config.json npm run start
 ```
 
 When it's running, you should see output like this:

backend/package.json

@@ -22,7 +22,10 @@
   "main": "index.ts",
   "scripts": {
     "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
-    "build": "npm run tsc",
+    "build": "npm run tsc && npm run create-resources",
+    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
+    "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=4096 dist/index.js",
     "test": "./node_modules/.bin/jest --coverage",

backend/src/api/backend-info.ts

@@ -1,60 +1,37 @@
-import * as fs from 'fs';
-import * as os from 'os';
-import logger from '../logger';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
-const { spawnSync } = require('child_process');
 
 class BackendInfo {
-  private gitCommitHash = '';
-  private hostname = '';
-  private version = '';
+  private backendInfo: IBackendInfo;
 
   constructor() {
-    this.setLatestCommitHash();
-    this.setVersion();
-    this.hostname = os.hostname();
-  }
-
-  public getBackendInfo(): IBackendInfo {
-    return {
-      hostname: this.hostname,
-      gitCommit: this.gitCommitHash,
-      version: this.version,
+    // This file is created by ./fetch-version.ts during building
+    const versionFile = path.join(__dirname, 'version.json')
+    var versionInfo;
+    if (fs.existsSync(versionFile)) {
+      versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
+    } else {
+      // Use dummy values if `versionFile` doesn't exist (e.g., during testing)
+      versionInfo = {
+        version: '?',
+        gitCommit: '?'
+      };
+    }
+    this.backendInfo = {
+      hostname: os.hostname(),
+      version: versionInfo.version,
+      gitCommit: versionInfo.gitCommit
     };
   }
 
+  public getBackendInfo(): IBackendInfo {
+    return this.backendInfo;
+  }
+
   public getShortCommitHash() {
-    return this.gitCommitHash.slice(0, 7);
-  }
-
-  private setLatestCommitHash(): void {
-    //TODO: share this logic with `generate-config.js`
-    if (process.env.DOCKER_COMMIT_HASH) {
-      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
-    } else {
-      try {
-        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
-        if (!gitRevParse.error) {
-          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
-          this.gitCommitHash = output ? output : '?';
-        } else if (gitRevParse.error.code === 'ENOENT') {
-          console.log('git not found, cannot parse git hash');
-          this.gitCommitHash = '?';
-        }
-      } catch (e: any) {
-        console.log('Could not load git commit info: ' + e.message);
-        this.gitCommitHash = '?';
-      }
-    }
-  }
-
-  private setVersion(): void {
-    try {
-      const packageJson = fs.readFileSync('package.json').toString();
-      this.version = JSON.parse(packageJson).version;
-    } catch (e) {
-      throw new Error(e instanceof Error ? e.message : 'Error');
-    }
+    return this.backendInfo.gitCommit.slice(0, 7);
   }
 }
 

backend/src/api/fetch-version.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from "path";
+const { spawnSync } = require('child_process');
+
+function getVersion(): string {
+  const packageJson = fs.readFileSync('package.json').toString();
+  return JSON.parse(packageJson).version;
+}
+
+function getGitCommit(): string {
+  if (process.env.MEMPOOL_COMMIT_HASH) {
+    return process.env.MEMPOOL_COMMIT_HASH;
+  } else {
+    const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+    if (!gitRevParse.error) {
+      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+      if (output) {
+        return output;
+      } else {
+        console.log('Could not fetch git commit: No repo available');
+      }
+    } else if (gitRevParse.error.code === 'ENOENT') {
+      console.log('Could not fetch git commit: Command `git` is unavailable');
+    }
+  }
+  return '?';
+}
+
+const versionInfo = {
+  version: getVersion(),
+  gitCommit: getGitCommit()
+}
+
+fs.writeFileSync(
+  path.join(__dirname, 'version.json'),
+  JSON.stringify(versionInfo, null, 2) + "\n"
+);

backend/src/config.ts

@@ -1,4 +1,6 @@
-const configFile = require('../mempool-config.json');
+const configFromFile = require(
+    process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
+);
 
 interface IConfig {
   MEMPOOL: {
@@ -249,7 +251,7 @@ class Config implements IConfig {
   MAXMIND: IConfig['MAXMIND'];
 
   constructor() {
-    const configs = this.merge(configFile, defaults);
+    const configs = this.merge(configFromFile, defaults);
     this.MEMPOOL = configs.MEMPOOL;
     this.ESPLORA = configs.ESPLORA;
     this.ELECTRUM = configs.ELECTRUM;

backend/src/tasks/price-updater.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import path from "path";
 import { Common } from '../api/common';
 import config from '../config';
 import logger from '../logger';
@@ -159,7 +160,7 @@ class PriceUpdater {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
     // Insert MtGox weekly prices
-    const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
+    const pricesJson: any[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'mtgox-weekly.json')).toString());
     const prices = this.getEmptyPricesObj();
     let insertedCount: number = 0;
     for (const price of pricesJson) {

docker/backend/Dockerfile

@@ -1,7 +1,7 @@
 FROM node:16.16.0-buster-slim AS builder
 
 ARG commitHash
-ENV DOCKER_COMMIT_HASH=${commitHash}
+ENV MEMPOOL_COMMIT_HASH=${commitHash}
 
 WORKDIR /build
 COPY . .
@@ -9,18 +9,15 @@ COPY . .
 RUN apt-get update
 RUN apt-get install -y build-essential python3 pkg-config
 RUN npm install --omit=dev --omit=optional
-RUN npm run build
+RUN npm run package
 
 FROM node:16.16.0-buster-slim
 
 WORKDIR /backend
 
-COPY --from=builder /build/ .
-
-RUN chmod +x /backend/start.sh
-RUN chmod +x /backend/wait-for-it.sh
-
-RUN chown -R 1000:1000 /backend && chmod -R 755 /backend
+RUN chown 1000:1000 ./
+COPY --from=builder --chown=1000:1000 /build/package ./package/
+COPY --from=builder --chown=1000:1000 /build/mempool-config.json /build/start.sh /build/wait-for-it.sh ./
 
 USER 1000
 

docker/backend/start.sh

@@ -205,4 +205,4 @@ sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
 # CLN
 sed -i "s!__CLN_SOCKET__!${__CLN_SOCKET__}!g" mempool-config.json
 
-node /backend/dist/index.js
+node /backend/package/index.js

docker/init.sh

@@ -1,10 +1,7 @@
 #!/bin/sh
 
 #backend
-gitMaster="\.\.\/\.git\/refs\/heads\/master"
-git ls-remote https://github.com/mempool/mempool.git "$1^{}" | awk '{ print $1}' > ./backend/master
 cp ./docker/backend/* ./backend/
-sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts
 
 #frontend
 localhostIP="127.0.0.1"