diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..a2eaeb495 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI Pipeline for the Backend and Frontend +on: + push: +env: + NODE_VERSION: 16.15.0 +jobs: + build_backend: + name: Build backend + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: 'https://registry.npmjs.org' + - name: Install + run: npm install --prod + working-directory: backend + # - name: Lint + # run: npm run lint + # - name: Test + # run: npm run test + - name: Build + run: npm run build + working-directory: backend + build_frontend: + name: Build frontend + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: 'https://registry.npmjs.org' + - name: Install + run: npm install --prod + working-directory: frontend + # - name: Lint + # run: npm run lint + # - name: Test + # run: npm run test + - name: Build + run: npm run build + working-directory: frontend diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 369b101ff..b057e6141 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -1,8 +1,11 @@ name: Cypress Tests -on: [push, pull_request] +on: + pull_request: + types: [ opened, review_requested, synchronize ] jobs: cypress: + if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/backend/package-lock.json b/backend/package-lock.json index 378bb7c75..7f921f476 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,6 +10,7 @@ "license": "GNU Affero General Public License v3.0", "dependencies": { "@mempool/electrum-client": "^1.1.7", + "@types/node": "^16.11.41", "axios": "~0.27.2", "bitcoinjs-lib": "6.0.1", "crypto-js": "^4.0.0", @@ -119,10 +120,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "14.14.20", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.20.tgz", - "integrity": "sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A==", - "dev": true + "version": "16.11.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz", + "integrity": "sha512-mqoYK2TnVjdkGk8qXAVGc/x9nSaTpSrFaGFm43BUH3IdoBV0nta6hYaGmdOvIMlbHJbUEVen3gvwpwovAZKNdQ==" }, "node_modules/@types/qs": { "version": "6.9.7", @@ -1647,10 +1647,9 @@ "dev": true }, "@types/node": { - "version": "14.14.20", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.20.tgz", - "integrity": "sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A==", - "dev": true + "version": "16.11.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz", + "integrity": "sha512-mqoYK2TnVjdkGk8qXAVGc/x9nSaTpSrFaGFm43BUH3IdoBV0nta6hYaGmdOvIMlbHJbUEVen3gvwpwovAZKNdQ==" }, "@types/qs": { "version": "6.9.7", diff --git a/backend/package.json b/backend/package.json index 723029e34..09a61c7cc 100644 --- a/backend/package.json +++ b/backend/package.json @@ -20,7 +20,6 @@ ], "main": "index.ts", "scripts": { - "ng": "./node_modules/@angular/cli/bin/ng", "tsc": "./node_modules/typescript/bin/tsc", "build": "npm run tsc", "start": "node --max-old-space-size=2048 dist/index.js", @@ -29,6 +28,7 @@ }, "dependencies": { "@mempool/electrum-client": "^1.1.7", + "@types/node": "^16.11.41", "axios": "~0.27.2", "bitcoinjs-lib": "6.0.1", "crypto-js": "^4.0.0", @@ -41,8 +41,8 @@ }, "devDependencies": { "@types/compression": "^1.7.2", - "@types/ws": "~8.5.3", "@types/express": "^4.17.13", + "@types/ws": "~8.5.3", "tslint": "^6.1.0" } } diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index d6f2e1df8..54d75d67b 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -1,10 +1,10 @@ import config from '../../config'; +import Client from '@mempool/electrum-client'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import { IElectrumApi } from './electrum-api.interface'; import BitcoinApi from './bitcoin-api'; import logger from '../../logger'; -import * as ElectrumClient from '@mempool/electrum-client'; import * as sha256 from 'crypto-js/sha256'; import * as hexEnc from 'crypto-js/enc-hex'; import loadingIndicators from '../loading-indicators'; @@ -26,7 +26,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { onLog: (str) => { logger.debug(str); }, }; - this.electrumClient = new ElectrumClient( + this.electrumClient = new Client( config.ELECTRUM.PORT, config.ELECTRUM.HOST, config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp', @@ -35,7 +35,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { ); this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy) - .then(() => {}) + .then(() => { }) .catch((err) => { logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`); }); @@ -95,7 +95,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { async $getAddressTransactions(address: string, lastSeenTxId: string): Promise { const addressInfo = await this.bitcoindClient.validateAddress(address); if (!addressInfo || !addressInfo.isvalid) { - return []; + return []; } try { diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index fc185a31a..cf40d6952 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -1,6 +1,6 @@ import * as fs from 'fs'; const fsPromises = fs.promises; -import * as cluster from 'cluster'; +import cluster from 'cluster'; import memPool from './mempool'; import blocks from './blocks'; import logger from '../logger'; @@ -19,7 +19,7 @@ class DiskCache { constructor() { } async $saveCacheToDisk(): Promise { - if (!cluster.isMaster) { + if (!cluster.isPrimary) { return; } if (this.isWritingCache) { @@ -46,12 +46,12 @@ class DiskCache { blockSummaries: blocks.getBlockSummaries(), mempool: {}, mempoolArray: mempoolArray.splice(0, chunkSize), - }), {flag: 'w'}); + }), { flag: 'w' }); for (let i = 1; i < DiskCache.CHUNK_FILES; i++) { await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({ mempool: {}, mempoolArray: mempoolArray.splice(0, chunkSize), - }), {flag: 'w'}); + }), { flag: 'w' }); } logger.debug('Mempool and blocks data saved to disk cache'); this.isWritingCache = false; @@ -67,7 +67,7 @@ class DiskCache { fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString())); } } - + loadMempoolCache() { if (!fs.existsSync(DiskCache.FILE_NAME)) { return; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 9632b03e3..3c5b830cc 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,7 +1,9 @@ import logger from '../logger'; import * as WebSocket from 'ws'; -import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta, - OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces'; +import { + BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta, + OptimizedStatistic, ILoadingIndicators, IConversionRates +} from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; import backendInfo from './backend-info'; @@ -164,7 +166,7 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - this.wss.clients.forEach((client: WebSocket) => { + this.wss.clients.forEach((client) => { if (client.readyState !== WebSocket.OPEN) { return; } @@ -179,7 +181,7 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - this.wss.clients.forEach((client: WebSocket) => { + this.wss.clients.forEach((client) => { if (client.readyState !== WebSocket.OPEN) { return; } @@ -192,7 +194,7 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - this.wss.clients.forEach((client: WebSocket) => { + this.wss.clients.forEach((client) => { if (client.readyState !== WebSocket.OPEN) { return; } @@ -224,7 +226,7 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - this.wss.clients.forEach((client: WebSocket) => { + this.wss.clients.forEach((client) => { if (client.readyState !== WebSocket.OPEN) { return; } @@ -255,7 +257,7 @@ class WebsocketHandler { memPool.handleRbfTransactions(rbfTransactions); const recommendedFees = feeApi.getRecommendedFee(); - this.wss.clients.forEach(async (client: WebSocket) => { + this.wss.clients.forEach(async (client) => { if (client.readyState !== WebSocket.OPEN) { return; } diff --git a/backend/src/index.ts b/backend/src/index.ts index 1b7568d2e..566622055 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,8 +1,8 @@ -import { Express, Request, Response, NextFunction } from 'express'; -import * as express from 'express'; +import express from "express"; +import { Application, Request, Response, NextFunction, Express } from 'express'; import * as http from 'http'; import * as WebSocket from 'ws'; -import * as cluster from 'cluster'; +import cluster from 'cluster'; import axios from 'axios'; import DB from './database'; @@ -32,7 +32,7 @@ import priceUpdater from './tasks/price-updater'; class Server { private wss: WebSocket.Server | undefined; private server: http.Server | undefined; - private app: Express; + private app: Application; private currentBackendRetryInterval = 5; constructor() { @@ -43,7 +43,7 @@ class Server { return; } - if (cluster.isMaster) { + if (cluster.isPrimary) { logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`); const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS; @@ -77,7 +77,7 @@ class Server { }) .use(express.urlencoded({ extended: true })) .use(express.text()) - ; + ; this.server = http.createServer(this.app); this.wss = new WebSocket.Server({ server: this.server }); @@ -105,7 +105,7 @@ class Server { } } - if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) { + if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) { statistics.startStatistics(); } @@ -260,7 +260,7 @@ class Server { res.status(500).end(); } }) - ; + ; if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) { this.app @@ -290,7 +290,7 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight) - ; + ; } if (config.BISQ.ENABLED) { @@ -340,7 +340,7 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix) - ; + ; } if (Common.isLiquid()) { @@ -349,13 +349,13 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets) .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon) .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup) - ; + ; } if (Common.isLiquid() && config.DATABASE.ENABLED) { this.app .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth) - ; + ; } } } diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 8b4cbea2e..6aa69dc08 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "types": ["node"], "module": "commonjs", "target": "esnext", "lib": ["es2019", "dom"], @@ -11,7 +12,8 @@ "typeRoots": [ "node_modules/@types" ], - "allowSyntheticDefaultImports": true + "allowSyntheticDefaultImports": true, + "esModuleInterop": true }, "include": [ "src/**/*.ts" diff --git a/frontend/package.json b/frontend/package.json index e9463d88e..25b9808c6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,20 +20,20 @@ ], "main": "index.ts", "scripts": { - "ng": "./node_modules/@angular/cli/bin/ng", + "ng": "./node_modules/@angular/cli/bin/ng.js", "tsc": "./node_modules/typescript/bin/tsc", "i18n-extract-from-source": "./node_modules/@angular/cli/bin/ng extract-i18n --out-file ./src/locale/messages.xlf", "i18n-pull-from-transifex": "tx pull -a --parallel --minimum-perc 1 --force", - "serve": "npm run generate-config && ng serve -c local", - "serve:stg": "npm run generate-config && ng serve -c staging", - "serve:local-prod": "npm run generate-config && ng serve -c local-prod", - "serve:local-staging": "npm run generate-config && ng serve -c local-staging", - "start": "npm run generate-config && npm run sync-assets-dev && ng serve -c local", - "start:stg": "npm run generate-config && npm run sync-assets-dev && ng serve -c staging", - "start:local-prod": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-prod", - "start:local-staging": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-staging", - "start:mixed": "npm run generate-config && npm run sync-assets-dev && ng serve -c mixed", - "build": "npm run generate-config && ng build --configuration production --localize && npm run sync-assets && npm run build-mempool.js", + "serve": "npm run generate-config && npm run ng -- serve -c local", + "serve:stg": "npm run generate-config && npm run ng -- serve -c staging", + "serve:local-prod": "npm run generate-config && npm run ng -- serve -c local-prod", + "serve:local-staging": "npm run generate-config && npm run ng -- serve -c local-staging", + "start": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local", + "start:stg": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c staging", + "start:local-prod": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-prod", + "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging", + "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed", + "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js", "sync-assets": "node sync-assets.js && rsync -av ./dist/mempool/browser/en-US/resources ./dist/mempool/browser/resources", "sync-assets-dev": "node sync-assets.js dev", "generate-config": "node generate-config.js", @@ -41,17 +41,17 @@ "build-mempool-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index.js --standalone mempoolJS > ./dist/mempool/browser/en-US/mempool.js", "build-mempool-bisq-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-bisq.js --standalone bisqJS > ./dist/mempool/browser/en-US/bisq.js", "build-mempool-liquid-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-liquid.js --standalone liquidJS > ./dist/mempool/browser/en-US/liquid.js", - "test": "ng test", - "lint": "ng lint", - "e2e": "npm run generate-config && ng e2e", + "test": "npm run ng -- test", + "lint": "npm run ng -- lint", + "e2e": "npm run generate-config && npm run ng -- e2e", "e2e:ci": "npm run cypress:run:ci", "config:defaults:mempool": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=mempool BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config", "config:defaults:liquid": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=liquid BLOCK_WEIGHT_UNITS=300000 && npm run generate-config", "config:defaults:bisq": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=bisq BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config", - "dev:ssr": "npm run generate-config && ng run mempool:serve-ssr", + "dev:ssr": "npm run generate-config && npm run ng -- run mempool:serve-ssr", "serve:ssr": "node server.run.js", - "build:ssr": "npm run build && ng run mempool:server:production && ./node_modules/typescript/bin/tsc server.run.ts", - "prerender": "ng run mempool:prerender", + "build:ssr": "npm run build && npm run ng -- run mempool:server:production && npm run tsc -- server.run.ts", + "prerender": "npm run ng -- run mempool:prerender", "cypress:open": "cypress open", "cypress:run": "cypress run", "cypress:run:record": "cypress run --record", diff --git a/production/bitcoin.conf b/production/bitcoin.conf index 9a46a63ad..4cb95eacc 100644 --- a/production/bitcoin.conf +++ b/production/bitcoin.conf @@ -22,6 +22,8 @@ bind=0.0.0.0:8333 bind=[::]:8333 rpcbind=127.0.0.1:8332 rpcbind=[::1]:8332 +zmqpubrawblock=tcp://127.0.0.1:18332 +zmqpubrawtx=tcp://127.0.0.1:18333 #addnode=[2401:b140:2::92:201]:8333 #addnode=[2401:b140:2::92:202]:8333 #addnode=[2401:b140:2::92:203]:8333 diff --git a/production/install b/production/install index 5197ce62f..2ab25ac97 100755 --- a/production/install +++ b/production/install @@ -282,7 +282,7 @@ BITCOIN_REPO_URL=https://github.com/bitcoin/bitcoin BITCOIN_REPO_NAME=bitcoin BITCOIN_REPO_BRANCH=master #BITCOIN_LATEST_RELEASE=$(curl -s https://api.github.com/repos/bitcoin/bitcoin/releases/latest|grep tag_name|head -1|cut -d '"' -f4) -BITCOIN_LATEST_RELEASE=v22.0 +BITCOIN_LATEST_RELEASE=v23.0 echo -n '.' BISQ_REPO_URL=https://github.com/bisq-network/bisq @@ -330,7 +330,7 @@ DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python-certbot-nginx rsync ufw # packages needed for mempool ecosystem FREEBSD_PKG=() FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim) -FREEBSD_PKG+=(openssh-portable py38-pip rust llvm90 jq base64) +FREEBSD_PKG+=(openssh-portable py38-pip rust llvm90 jq base64 libzmq4) FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf) FREEBSD_PKG+=(nginx rsync py38-certbot-nginx mariadb105-server keybase)