Merge pull request #4078 from mempool/fix/difficulty-calc

Fix: Use bits to calculate difficulty instead of floating points

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -1,4 +1,8 @@
-import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+import {
+  calcBitsDifference,
+  calcDifficultyAdjustment,
+  DifficultyAdjustment,
+} from '../../api/difficulty-adjustment';
 
 describe('Mempool Difficulty Adjustment', () => {
   test('should calculate Difficulty Adjustments properly', () => {
@@ -86,4 +90,46 @@ describe('Mempool Difficulty Adjustment', () => {
       expect(result).toStrictEqual(vector[1]);
     }
   });
+
+  test('should calculate Difficulty change from bits fields of two blocks', () => {
+    // Check same exponent + check min max for output
+    expect(calcBitsDifference(0x1d000200, 0x1d000100)).toEqual(100);
+    expect(calcBitsDifference(0x1d000400, 0x1d000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000800, 0x1d000100)).toEqual(300); // Actually 700
+    expect(calcBitsDifference(0x1d000100, 0x1d000200)).toEqual(-50);
+    expect(calcBitsDifference(0x1d000100, 0x1d000400)).toEqual(-75);
+    expect(calcBitsDifference(0x1d000100, 0x1d000800)).toEqual(-75); // Actually -87.5
+    // Check new higher exponent
+    expect(calcBitsDifference(0x1c000200, 0x1d000001)).toEqual(100);
+    expect(calcBitsDifference(0x1c000400, 0x1d000001)).toEqual(300);
+    expect(calcBitsDifference(0x1c000800, 0x1d000001)).toEqual(300);
+    expect(calcBitsDifference(0x1c000100, 0x1d000002)).toEqual(-50);
+    expect(calcBitsDifference(0x1c000100, 0x1d000004)).toEqual(-75);
+    expect(calcBitsDifference(0x1c000100, 0x1d000008)).toEqual(-75);
+    // Check new lower exponent
+    expect(calcBitsDifference(0x1d000002, 0x1c000100)).toEqual(100);
+    expect(calcBitsDifference(0x1d000004, 0x1c000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000008, 0x1c000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000001, 0x1c000200)).toEqual(-50);
+    expect(calcBitsDifference(0x1d000001, 0x1c000400)).toEqual(-75);
+    expect(calcBitsDifference(0x1d000001, 0x1c000800)).toEqual(-75);
+    // Check error when exponents are too far apart
+    expect(() => calcBitsDifference(0x1d000001, 0x1a000800)).toThrow(
+      /Impossible exponent difference/
+    );
+    // Check invalid inputs
+    expect(() => calcBitsDifference(0x7f000001, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0, 0x1a000800)).toThrow(/Invalid bits/);
+    expect(() => calcBitsDifference(100.2783, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0x00800000, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0x1c000000, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+  });
 });

backend/src/api/blocks.ts

@@ -28,12 +28,13 @@ import chainTips from './chain-tips';
 import websocketHandler from './websocket-handler';
 import redisCache from './redis-cache';
 import rbfCache from './rbf-cache';
+import { calcBitsDifference } from './difficulty-adjustment';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
   private blockSummaries: BlockSummary[] = [];
   private currentBlockHeight = 0;
-  private currentDifficulty = 0;
+  private currentBits = 0;
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
@@ -666,14 +667,14 @@ class Blocks {
         const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
         this.updateTimerProgress(timer, 'got block for initial difficulty adjustment');
         this.lastDifficultyAdjustmentTime = block.timestamp;
-        this.currentDifficulty = block.difficulty;
+        this.currentBits = block.bits;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
           this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
           const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
           this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
-          this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
+          this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits);
           logger.debug(`Initial difficulty adjustment data set.`);
         }
       } else {
@@ -786,14 +787,18 @@ class Blocks {
             time: block.timestamp,
             height: block.height,
             difficulty: block.difficulty,
-            adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
+            adjustment: Math.round(
+              // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio.
+              // Instead of actually doing /100, just reduce the multiplier.
+              (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000
+            ) / 1000000, // Remove float point noise
           });
           this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
         }
 
-        this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
+        this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits);
         this.lastDifficultyAdjustmentTime = block.timestamp;
-        this.currentDifficulty = block.difficulty;
+        this.currentBits = block.bits;
       }
 
       // wait for pending async callbacks to finish

backend/src/api/difficulty-adjustment.ts

@@ -16,6 +16,68 @@ export interface DifficultyAdjustment {
   expectedBlocks: number;         // Block count
 }
 
+/**
+ * Calculate the difficulty increase/decrease by using the `bits` integer contained in two
+ * block headers.
+ *
+ * Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code
+ * assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an
+ * error if an exponent difference of 2 or more is seen.
+ *
+ * @param {number} oldBits The 32 bit `bits` integer from a block header.
+ * @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period.
+ * @returns {number} A floating point decimal of the difficulty change from old to new.
+ *          (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty)
+ */
+export function calcBitsDifference(oldBits: number, newBits: number): number {
+  // Must be
+  // - integer
+  // - highest exponent is 0x1f, so max value (as integer) is 0x1f0000ff
+  // - min value is 1 (exponent = 0)
+  // - highest bit of the number-part is +- sign, it must not be 1
+  const verifyBits = (bits: number): void => {
+    if (
+      Math.floor(bits) !== bits ||
+      bits > 0x1f0000ff ||
+      bits < 1 ||
+      (bits & 0x00800000) !== 0 ||
+      (bits & 0x007fffff) === 0
+    ) {
+      throw new Error('Invalid bits');
+    }
+  };
+  verifyBits(oldBits);
+  verifyBits(newBits);
+
+  // No need to mask exponents because we checked the bounds above
+  const oldExp = oldBits >> 24;
+  const newExp = newBits >> 24;
+  const oldNum = oldBits & 0x007fffff;
+  const newNum = newBits & 0x007fffff;
+  // The diff can only possibly be 1, 0, -1
+  // (because maximum difficulty change is x4 or /4 (2 bits up or down))
+  let result: number;
+  switch (newExp - oldExp) {
+    // New less than old, target lowered, difficulty increased
+    case -1:
+      result = ((oldNum << 8) * 100) / newNum - 100;
+      break;
+    // Same exponent, compare numbers as is.
+    case 0:
+      result = (oldNum * 100) / newNum - 100;
+      break;
+    // Old less than new, target raised, difficulty decreased
+    case 1:
+      result = (oldNum * 100) / (newNum << 8) - 100;
+      break;
+    default:
+      throw new Error('Impossible exponent difference');
+  }
+
+  // Min/Max values
+  return result > 300 ? 300 : result < -75 ? -75 : result;
+}
+
 export function calcDifficultyAdjustment(
   DATime: number,
   nowSeconds: number,