lightningd: handle bcli plugins returning fee_floor and feerates parameters.

Changelog-Added: Plugins: `estimatefees` can return explicit `fee_floor` and `feerates` by block number.
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

doc/PLUGINS.md

@@ -1729,17 +1729,26 @@ The plugin must respond to `getchaininfo` with the following fields:
 
 Polled by `lightningd` to get the current feerate, all values must be passed in sat/kVB.
 
-If fee estimation fails, the plugin must set all the fields to `null`.
+The plugin must return `feerate_floor` (e.g. 1000 if mempool is
+empty), and an array of 0 or more `feerates`.  Each element of
+`feerates` is an object with `blocks` and `feerate`, in
+ascending-blocks order, for example:
 
-The plugin, if fee estimation succeeds, must respond with the following fields:
+```
-    - `opening` (number), used for funding and also misc transactions
+{
-    - `mutual_close` (number), used for the mutual close transaction
+	"feerate_floor": <sat per kVB>,
-    - `unilateral_close` (number), used for unilateral close (/commitment) transactions
+	"feerates": {
-    - `delayed_to_us` (number), used for resolving our output from our unilateral close
+		{ "blocks": 2, "feerate": <sat per kVB> },
-    - `htlc_resolution` (number), used for resolving HTLCs after an unilateral close
+		{ "blocks": 6, "feerate": <sat per kVB> },
-    - `penalty` (number), used for resolving revoked transactions
+		{ "blocks": 12, "feerate": <sat per kVB> }
-    - `min_acceptable` (number), used as the minimum acceptable feerate
+		{ "blocks": 100, "feerate": <sat per kVB> }
-    - `max_acceptable` (number), used as the maximum acceptable feerate
+	}
+}
+```
+
+lightningd will currently linearly interpolate to estimate between
+given blocks (it will not extrapolate, but use the min/max blocks
+values).
 
 
 ### `getrawblockbyheight`

lightningd/bitcoind.c

@@ -12,6 +12,7 @@
 #include <ccan/array_size/array_size.h>
 #include <ccan/io/io.h>
 #include <ccan/tal/str/str.h>
+#include <common/configdir.h>
 #include <common/json_parse.h>
 #include <common/memleak.h>
 #include <db/exec.h>
@@ -144,7 +145,7 @@ static void bitcoin_plugin_send(struct bitcoind *bitcoind,
  *   - `min` is the minimum acceptable feerate
  *   - `max` is the maximum acceptable feerate
  *
- * Plugin response:
+ * Plugin response (deprecated):
  * {
  *	"opening": <sat per kVB>,
  *	"mutual_close": <sat per kVB>,
@@ -155,6 +156,19 @@ static void bitcoin_plugin_send(struct bitcoind *bitcoind,
  *	"min_acceptable": <sat per kVB>,
  *	"max_acceptable": <sat per kVB>,
  * }
+ *
+ * Plugin response (modern):
+ * {
+ *	"feerate_floor": <sat per kVB>,
+ *	"feerates": {
+ *		{ "blocks": 2, "feerate": <sat per kVB> },
+ *		{ "blocks": 6, "feerate": <sat per kVB> },
+ *		{ "blocks": 12, "feerate": <sat per kVB> }
+ *		{ "blocks": 100, "feerate": <sat per kVB> }
+ *	}
+ * }
+ *
+ * If rates are missing, we linearly interpolate (we don't extrapolate tho!).
  */
 struct estimatefee_call {
 	struct bitcoind *bitcoind;
@@ -163,6 +177,66 @@ struct estimatefee_call {
 };
 
 /* Note: returns estimates in perkb, caller converts! */
+static struct feerate_est *parse_feerate_ranges(const tal_t *ctx,
+						struct bitcoind *bitcoind,
+						const char *buf,
+						const jsmntok_t *floortok,
+						const jsmntok_t *feerates,
+						u32 *floor)
+{
+	size_t i;
+	const jsmntok_t *t;
+	struct feerate_est *rates = tal_arr(ctx, struct feerate_est, 0);
+
+	if (!json_to_u32(buf, floortok, floor))
+		bitcoin_plugin_error(bitcoind, buf, floortok,
+				     "estimatefees.feerate_floor", "Not a u32?");
+
+	json_for_each_arr(i, t, feerates) {
+		struct feerate_est rate;
+		const char *err;
+
+		err = json_scan(tmpctx, buf, t, "{blocks:%,feerate:%}",
+				JSON_SCAN(json_to_u32, &rate.blockcount),
+				JSON_SCAN(json_to_u32, &rate.rate));
+		if (err)
+			bitcoin_plugin_error(bitcoind, buf, t,
+					     "estimatefees.feerates", err);
+
+		/* Block count must be in order.  If rates go up somehow, we
+		 * reduce to prev. */
+		if (tal_count(rates) != 0) {
+			const struct feerate_est *prev = &rates[tal_count(rates)-1];
+			if (rate.blockcount <= prev->blockcount)
+				bitcoin_plugin_error(bitcoind, buf, feerates,
+						     "estimatefees.feerates",
+						     "Blocks must be ascending"
+						     " order: %u <= %u!",
+						     rate.blockcount,
+						     prev->blockcount);
+			if (rate.rate > prev->rate) {
+				log_unusual(bitcoind->log,
+					    "Feerate for %u blocks (%u) is > rate"
+					    " for %u blocks (%u)!",
+					    rate.blockcount, rate.rate,
+					    prev->blockcount, prev->rate);
+				rate.rate = prev->rate;
+			}
+		}
+
+		tal_arr_expand(&rates, rate);
+	}
+
+	if (tal_count(rates) == 0) {
+		if (chainparams->testnet)
+			log_debug(bitcoind->log, "Unable to estimate any fees");
+		else
+			log_unusual(bitcoind->log, "Unable to estimate any fees");
+	}
+
+	return rates;
+}
+
 static struct feerate_est *parse_deprecated_feerates(const tal_t *ctx,
 						     struct bitcoind *bitcoind,
 						     const char *buf,
@@ -216,7 +290,7 @@ static void estimatefees_callback(const char *buf, const jsmntok_t *toks,
 				  const jsmntok_t *idtok,
 				  struct estimatefee_call *call)
 {
-	const jsmntok_t *resulttok;
+	const jsmntok_t *resulttok, *floortok;
 	struct feerate_est *feerates;
 	u32 floor;
 
@@ -226,10 +300,19 @@ static void estimatefees_callback(const char *buf, const jsmntok_t *toks,
 				     "estimatefees",
 				     "bad 'result' field");
 
-	feerates = parse_deprecated_feerates(call, call->bitcoind,
+	/* Modern style has floor. */
-					     buf, resulttok);
+	floortok = json_get_member(buf, resulttok, "feerate_floor");
-	/* FIXME: get from plugin! */
+	if (floortok) {
-	floor = feerate_from_style(FEERATE_FLOOR, FEERATE_PER_KSIPA);
+		feerates = parse_feerate_ranges(call, call->bitcoind,
+						buf, floortok,
+						json_get_member(buf, resulttok,
+								"feerates"),
+						&floor);
+	} else {
+		feerates = parse_deprecated_feerates(call, call->bitcoind,
+						     buf, resulttok);
+		floor = feerate_from_style(FEERATE_FLOOR, FEERATE_PER_KSIPA);
+	}
 
 	/* Convert to perkw */
 	floor = feerate_from_style(floor, FEERATE_PER_KBYTE);

lightningd/chaintopology.c

@@ -610,8 +610,7 @@ u32 penalty_feerate(struct chain_topology *topo)
 
 u32 get_feerate_floor(const struct chain_topology *topo)
 {
-	/* FIXME: Make this dynamic! */
+	return topo->feerate_floor;
-	return FEERATE_FLOOR;
 }
 
 static struct command_result *json_feerates(struct command *cmd,

tests/test_misc.py

@@ -1942,9 +1942,8 @@ def test_bitcoind_fail_first(node_factory, bitcoind):
 
 
 @unittest.skipIf(TEST_NETWORK == 'liquid-regtest', "Fees on elements are different")
-@unittest.skip("FIXME: temporarily broken")
 def test_bitcoind_feerate_floor(node_factory, bitcoind):
-    """Don't return a feerate less than minrelaytxfee/mempoolnifee."""
+    """Don't return a feerate less than minrelaytxfee/mempoolminfee."""
     l1 = node_factory.get_node()
 
     anchors = EXPERIMENTAL_FEATURES
@@ -1953,11 +1952,21 @@ def test_bitcoind_feerate_floor(node_factory, bitcoind):
             "opening": 30000,
             "mutual_close": 15000,
             "unilateral_close": 44000,
-            "delayed_to_us": 30000,
-            "htlc_resolution": 44000,
             "penalty": 30000,
             "min_acceptable": 7500,
-            "max_acceptable": 600000
+            "max_acceptable": 600000,
+            "estimates": [{"blockcount": 2,
+                           "feerate": 60000,
+                           "smoothed_feerate": 60000},
+                          {"blockcount": 6,
+                           "feerate": 44000,
+                           "smoothed_feerate": 44000},
+                          {"blockcount": 12,
+                           "feerate": 30000,
+                           "smoothed_feerate": 30000},
+                          {"blockcount": 100,
+                           "feerate": 15000,
+                           "smoothed_feerate": 15000}],
         },
         "onchain_fee_estimates": {
             "opening_channel_satoshis": 5265,
@@ -1980,12 +1989,22 @@ def test_bitcoind_feerate_floor(node_factory, bitcoind):
             # This has increased (rounded up)
             "mutual_close": 20004,
             "unilateral_close": 44000,
-            "delayed_to_us": 30000,
-            "htlc_resolution": 44000,
             "penalty": 30000,
-            # This has increased (rounded up!)
+            # FIXME: this should increase:
-            "min_acceptable": 20004,
+            "min_acceptable": 10000,
-            "max_acceptable": 600000
+            "max_acceptable": 600000,
+            "estimates": [{"blockcount": 2,
+                           "feerate": 60000,
+                           "smoothed_feerate": 60000},
+                          {"blockcount": 6,
+                           "feerate": 44000,
+                           "smoothed_feerate": 44000},
+                          {"blockcount": 12,
+                           "feerate": 30000,
+                           "smoothed_feerate": 30000},
+                          {"blockcount": 100,
+                           "feerate": 20004,
+                           "smoothed_feerate": 20004}],
         },
         "onchain_fee_estimates": {
             "opening_channel_satoshis": 5265,
@@ -2011,13 +2030,24 @@ def test_bitcoind_feerate_floor(node_factory, bitcoind):
             "mutual_close": 30004,
             "unilateral_close": 44000,
             # This has increased (rounded up!)
-            "delayed_to_us": 30004,
-            "htlc_resolution": 44000,
-            # This has increased (rounded up!)
             "penalty": 30004,
-            # This has increased (rounded up!)
+            # FIXME: this should increase to 30004!
-            "min_acceptable": 30004,
+            "min_acceptable": 15000,
-            "max_acceptable": 600000
+            "max_acceptable": 600000,
+            "estimates": [{"blockcount": 2,
+                           "feerate": 60000,
+                           "smoothed_feerate": 60000},
+                          {"blockcount": 6,
+                           "feerate": 44000,
+                           "smoothed_feerate": 44000},
+                          # This has increased (rounded up!)
+                          {"blockcount": 12,
+                           "feerate": 30004,
+                           "smoothed_feerate": 30004},
+                          # This has increased (rounded up!)
+                          {"blockcount": 100,
+                           "feerate": 30004,
+                           "smoothed_feerate": 30004}],
         },
         "onchain_fee_estimates": {
             "opening_channel_satoshis": 5265,