TST: add retries for integration tests on CI

.circleci/config.yml

@@ -52,6 +52,9 @@ jobs:
     docker:
       - image: cimg/node:16.20.2
 
+    environment:
+      RETRY: "1"
+
     working_directory: ~/repo
 
     resource_class: large

.github/workflows/ci.yml

@@ -53,6 +53,7 @@ jobs:
         FAULTY_ZPUB: ${{ secrets.FAULTY_ZPUB }}
         MNEMONICS_COBO: ${{ secrets.MNEMONICS_COBO }}
         MNEMONICS_COLDCARD: ${{ secrets.MNEMONICS_COLDCARD }}
+        RETRY: 1
 
   e2e:
     runs-on: ubuntu-latest

package-lock.json

@@ -139,6 +139,7 @@
         "eslint-plugin-react": "^7.34.1",
         "eslint-plugin-react-native": "^4.1.0",
         "jest": "^29.4.2",
+        "jest-environment-node": "^29.7.0",
         "node-fetch": "^2.6.7",
         "prettier": "^3.2.5",
         "react-test-renderer": "18.2.0",

package.json

@@ -35,6 +35,7 @@
     "eslint-plugin-react": "^7.34.1",
     "eslint-plugin-react-native": "^4.1.0",
     "jest": "^29.4.2",
+    "jest-environment-node": "^29.7.0",
     "node-fetch": "^2.6.7",
     "prettier": "^3.2.5",
     "react-test-renderer": "18.2.0",
@@ -58,7 +59,7 @@
     "postinstall": "rn-nodeify --install buffer,events,process,stream,inherits,path,assert,crypto --hack; npm run releasenotes2json; npm run branch2json; npm run patches",
     "patches": "patch -p1   < scripts/rn-ldk.patch; patch -p1   < scripts/react-native-camera-kit.patch;",
     "test": "npm run tslint && npm run lint && npm run unit && npm run jest",
-    "jest": "jest -b tests/integration/*",
+    "jest": "jest tests/integration/*",
     "e2e:debug-build": "detox build -c android.debug",
     "e2e:debug-test": "detox test -c android.debug -d 200000 -l info",
     "e2e:debug": "(test -f android/app/build/outputs/apk/debug/app-debug.apk && test -f android/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk) || npm run e2e:debug-build; npm run e2e:debug-test",
@@ -68,9 +69,17 @@
     "lint": " npm run tslint && node scripts/find-unused-loc.js && eslint --ext .js,.ts,.tsx '*.@(js|ts|tsx)' screen 'blue_modules/*.@(js|ts|tsx)' class models loc tests components navigation typings",
     "lint:fix": "npm run lint -- --fix",
     "lint:quickfix": "git status --porcelain | grep -v '\\.json' | grep -E '\\.js|\\.ts' --color=never |  awk '{print $2}' | xargs eslint --fix; exit 0",
-    "unit": "jest -b -i tests/unit/*"
+    "unit": "jest -b tests/unit/*"
   },
   "jest": {
+    "testEnvironment": "<rootDir>/tests/custom-environment.js",
+    "reporters": [
+      "default",
+      [
+        "<rootDir>/tests/custom-reporter.js",
+        {}
+      ]
+    ],
     "preset": "react-native",
     "transform": {
       "^.+\\.(ts|tsx)$": "ts-jest"

tests/custom-environment.js

@@ -0,0 +1,17 @@
+import NodeEnvironment from 'jest-environment-node';
+class CustomEnvironment extends NodeEnvironment {
+  async handleTestEvent(event) {
+    if (event.name === 'test_start') {
+      if (!process.env.RETRY) return;
+
+      const fullName = (event.test.parent.name === 'ROOT_DESCRIBE_BLOCK' ? '' : event.test.parent.name + ' ') + event.test.name;
+      const hash = require('crypto').createHash('md5').update(fullName).digest('hex');
+      if (require('fs').existsSync(`/tmp/${hash}`)) {
+        event.test.mode = 'skip';
+        console.log('skipping as it previously passed on CI:', fullName);
+      }
+    }
+  }
+}
+
+module.exports = CustomEnvironment;

tests/custom-reporter.js

@@ -0,0 +1,34 @@
+/**
+ * @fileOverview to combat flakiness of jest integration tests we implement a diy retry mechanism:
+ * a custom reporter writes a lock file in /tmp for each successfull testcase.
+ * then when a test suite is restarted, a custom environment checks if a testcase passed previously and
+ * forcefully skips such test cases.
+ */
+class CustomReporter {
+  constructor(globalConfig, reporterOptions, reporterContext) {
+    this._globalConfig = globalConfig;
+    this._options = reporterOptions;
+    this._context = reporterContext;
+  }
+
+  onTestCaseResult(test, testCaseResult) {
+    if (!process.env.RETRY) return;
+
+    // since we cant distinguish several testcases in `it.each(...)`, we just ignore them so they will always run
+    if (testCaseResult.fullName.includes('can fetch balance, transactions & utxo, disableBatching=')) return;
+    if (testCaseResult.fullName.includes('BlueElectrum can do multiGetBalanceByAddress(), disableBatching=')) return;
+    if (testCaseResult.fullName.includes('ElectrumClient can do multiGetHistoryByAddress(), disableBatching=')) return;
+    if (testCaseResult.fullName.includes('ElectrumClient can do multiGetTransactionByTxid(), disableBatching=')) return;
+    if (testCaseResult.fullName.includes('ElectrumClient can do multiGetHistoryByAddress() to obtain txhex, disableBatching=')) return;
+    if (testCaseResult.fullName.includes('addresses for vout missing')) return;
+    if (testCaseResult.fullName.includes('txdatas were coming back null from BlueElectrum because of high batchsize')) return;
+
+    const hash = require('crypto').createHash('md5').update(testCaseResult.fullName).digest('hex');
+    if (testCaseResult.status === 'passed') {
+      // marking testcase as passed in /tmp
+      require('fs').writeFileSync(`/tmp/${hash}`, '1');
+    }
+  }
+}
+
+module.exports = CustomReporter;