upgradewallet: JSONRPC call to update p2sh outputs to a native segwit

v2 opens require you to use native segwit inputs

Changelog-Added: JSONRPC: `upgradewallet` command, sweeps all p2sh-wrapped outputs to a native segwit output

doc/index.rst

@@ -125,6 +125,7 @@ Core Lightning Documentation
    lightning-txprepare <lightning-txprepare.7.md>
    lightning-txsend <lightning-txsend.7.md>
    lightning-unreserveinputs <lightning-unreserveinputs.7.md>
+   lightning-upgradewallet <lightning-upgradewallet.7.md>
    lightning-utxopsbt <lightning-utxopsbt.7.md>
    lightning-waitanyinvoice <lightning-waitanyinvoice.7.md>
    lightning-waitblockheight <lightning-waitblockheight.7.md>

doc/lightning-upgradewallet.7.md

@@ -0,0 +1,58 @@
+lightning-upgradewallet -- Command to spend all P2SH-wrapped inputs into a Native Segwit output
+================================================================
+
+SYNOPSIS
+--------
+
+**upgradewallet** [*feerate*] [*reservedok*]
+
+DESCRIPTION
+-----------
+
+`upgradewallet` is a convenience RPC which will spend all p2sh-wrapped
+Segwit deposits in a wallet into a single Native Segwit P2WPKH address.
+
+*feerate* can be one of the feerates listed in lightning-feerates(7),
+or one of the strings *urgent* (aim for next block), *normal* (next 4
+blocks or so) or *slow* (next 100 blocks or so) to use lightningd's
+internal estimates.  It can also be a *feerate* is a number, with an
+optional suffix: *perkw* means the number is interpreted as
+satoshi-per-kilosipa (weight), and *perkb* means it is interpreted
+bitcoind-style as satoshi-per-kilobyte. Omitting the suffix is
+equivalent to *perkb*.
+
+*reservedok* tells the wallet to include all P2SH-wrapped inputs, including
+reserved ones.
+
+EXAMPLE USAGE
+-------------
+
+The caller is trying to buy a liquidity ad but the command keeps failing.
+They have funds in their wallet, but they're all P2SH-wrapped outputs.
+
+The caller can call `upgradewallet` to convert their funds to native segwit
+outputs, which are valid for liquidity ad buys.
+
+RETURN VALUE
+------------
+
+[comment]: # (GENERATE-FROM-SCHEMA-START)
+[comment]: # (GENERATE-FROM-SCHEMA-END)
+
+
+AUTHOR
+------
+
+~niftynei~ <<niftynei@gmail.com>> is mainly responsible.
+
+SEE ALSO
+--------
+
+lightning-utxopsbt(7), lightning-reserveinputs(7), lightning-unreserveinputs(7).
+
+RESOURCES
+---------
+
+Main web site: <https://github.com/ElementsProject/lightning>
+
+[comment]: # ( SHA256STAMP:0f290582f49c6103258b7f781a9e7fa4075ec6c05335a459a91da0b6fd58c68d)

doc/schemas/upgradewallet.request.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "required": [],
+  "additionalProperties": false,
+  "properties": {
+    "feerate": {
+      "type": "feerate",
+      "description": "Feerate for the upgrade transaction",
+      "added": "v23.02"
+    },
+    "reservedok": {
+      "type": "boolean",
+      "description": "Include already reserved funds or not",
+      "added": "v23.02"
+    }
+  }
+}

doc/schemas/upgradewallet.schema.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "upgraded_outs"
+  ],
+  "properties": {
+    "upgraded_outs": {
+      "type": "u64",
+      "description": "Count of spent/upgraded UTXOs",
+      "added": "v23.02"
+    },
+    "psbt": {
+      "type": "string",
+      "description": "The PSBT that was finalized and sent",
+      "added": "v23.02"
+    },
+    "tx": {
+      "type": "hex",
+      "description": "The raw transaction which was sent",
+      "added": "v23.02"
+    },
+    "txid": {
+      "type": "txid",
+      "description": "The txid of the **tx**",
+      "added": "v23.02"
+    }
+  }
+}

plugins/txprepare.c

@@ -35,6 +35,9 @@ struct txprepare {
 
 	/* For withdraw, we actually send immediately. */
 	bool is_withdraw;
+
+	/* Keep track if upgrade, so we can report on finish */
+	bool is_upgrade;
 };
 
 struct unreleased_tx {
@@ -42,6 +45,7 @@ struct unreleased_tx {
 	struct bitcoin_txid txid;
 	struct wally_tx *tx;
 	struct wally_psbt *psbt;
+	bool is_upgrade;
 };
 
 static LIST_HEAD(unreleased_txs);
@@ -137,6 +141,8 @@ static struct command_result *sendpsbt_done(struct command *cmd,
 	json_add_hex_talarr(out, "tx", linearize_wtx(tmpctx, utx->tx));
 	json_add_txid(out, "txid", &utx->txid);
 	json_add_psbt(out, "psbt", utx->psbt);
+	if (utx->is_upgrade)
+		json_add_num(out, "upgraded_outs", utx->tx->num_inputs);
 	return command_finished(cmd, out);
 }
 
@@ -208,6 +214,7 @@ static struct command_result *finish_txprepare(struct command *cmd,
 	psbt_elements_normalize_fees(txp->psbt);
 
 	utx = tal(NULL, struct unreleased_tx);
+	utx->is_upgrade = txp->is_upgrade;
 	utx->psbt = tal_steal(utx, txp->psbt);
 	psbt_txid(utx, txp->psbt, &utx->txid, &utx->tx);
 
@@ -351,7 +358,8 @@ static struct command_result *txprepare_continue(struct command *cmd,
 						 const char *feerate,
 						 unsigned int *minconf,
 						 struct bitcoin_outpoint *utxos,
-						 bool is_withdraw)
+						 bool is_withdraw,
+						 bool reservedok)
 {
 	struct out_req *req;
 
@@ -372,10 +380,12 @@ static struct command_result *txprepare_continue(struct command *cmd,
 			json_add_outpoint(req->js, NULL, &utxos[i]);
 		}
 		json_array_end(req->js);
+		json_add_bool(req->js, "reservedok", reservedok);
 	} else {
 		req = jsonrpc_request_start(cmd->plugin, cmd, "fundpsbt",
 					    psbt_created, forward_error,
 					    txp);
+		if (minconf)
 			json_add_u32(req->js, "minconf", *minconf);
 	}
 
@@ -407,7 +417,8 @@ static struct command_result *json_txprepare(struct command *cmd,
 		   NULL))
 		return command_param_failed();
 
-	return txprepare_continue(cmd, txp, feerate, minconf, utxos, false);
+	txp->is_upgrade = false;
+	return txprepare_continue(cmd, txp, feerate, minconf, utxos, false, false);
 }
 
 /* Called after we've unreserved the inputs. */
@@ -533,7 +544,151 @@ static struct command_result *json_withdraw(struct command *cmd,
 	txp->weight = bitcoin_tx_core_weight(1, tal_count(txp->outputs))
 		+ bitcoin_tx_output_weight(tal_bytelen(scriptpubkey));
 
-	return txprepare_continue(cmd, txp, feerate, minconf, utxos, true);
+	txp->is_upgrade = false;
+	return txprepare_continue(cmd, txp, feerate, minconf, utxos, true, false);
+}
+
+struct listfunds_info {
+	struct txprepare *txp;
+	const char *feerate;
+	bool reservedok;
+};
+
+/* Find all the utxos that are p2sh in our wallet */
+static struct command_result *listfunds_done(struct command *cmd,
+					     const char *buf,
+					     const jsmntok_t *result,
+					     struct listfunds_info *info)
+{
+	struct bitcoin_outpoint *utxos;
+	const jsmntok_t *outputs_tok, *tok;
+	size_t i;
+	struct txprepare *txp = info->txp;
+
+	/* Find all the utxos in our wallet that are p2sh! */
+	outputs_tok = json_get_member(buf, result, "outputs");
+	txp->output_total = AMOUNT_SAT(0);
+	if (!outputs_tok)
+		plugin_err(cmd->plugin,
+			   "`listfunds` payload has no outputs token: %*.s",
+			   json_tok_full_len(result),
+			   json_tok_full(buf, result));
+
+	utxos = tal_arr(cmd, struct bitcoin_outpoint, 0);
+	json_for_each_arr(i, tok, outputs_tok) {
+		struct bitcoin_outpoint prev_out;
+		struct amount_sat val;
+		bool is_reserved;
+		char *status;
+		const char *err;
+
+		err = json_scan(tmpctx, buf, tok,
+				"{amount_msat:%"
+				",status:%"
+				",reserved:%"
+				",txid:%"
+				",output:%}",
+				JSON_SCAN(json_to_sat, &val),
+				JSON_SCAN_TAL(cmd, json_strdup, &status),
+				JSON_SCAN(json_to_bool, &is_reserved),
+				JSON_SCAN(json_to_txid, &prev_out.txid),
+				JSON_SCAN(json_to_number, &prev_out.n));
+		if (err)
+			plugin_err(cmd->plugin,
+				   "`listfunds` payload did not scan. %s: %*.s",
+				   err, json_tok_full_len(result),
+				   json_tok_full(buf, result));
+
+		/* Skip non-p2sh outputs */
+		if (!json_get_member(buf, tok, "redeemscript"))
+			continue;
+
+		/* only include confirmed + unconfirmed outputs */
+		if (!streq(status, "confirmed")
+		    && !streq(status, "unconfirmed"))
+			continue;
+
+		if (!info->reservedok && is_reserved)
+			continue;
+
+		tal_arr_expand(&utxos, prev_out);
+	}
+
+	/* Nothing found to upgrade, return a success */
+	if (tal_count(utxos) == 0) {
+		struct json_stream *out;
+		out = jsonrpc_stream_success(cmd);
+		json_add_num(out, "upgraded_outs", tal_count(utxos));
+		return command_finished(cmd, out);
+	}
+
+	return txprepare_continue(cmd, txp, info->feerate,
+				  NULL, utxos, true,
+				  info->reservedok);
+}
+
+/* We've got an address for sending funds */
+static struct command_result *newaddr_sweep_done(struct command *cmd,
+						 const char *buf,
+						 const jsmntok_t *result,
+						 struct listfunds_info *info)
+{
+	struct out_req *req;
+	const jsmntok_t *addr = json_get_member(buf, result, "bech32");
+
+	info->txp = tal(info, struct txprepare);
+	info->txp->is_upgrade = true;
+
+	/* Add output for 'all' to txp */
+	info->txp->outputs = tal_arr(info->txp, struct tx_output, 1);
+	info->txp->all_output_idx = 0;
+	info->txp->output_total = AMOUNT_SAT(0);
+	info->txp->outputs[0].amount = AMOUNT_SAT(-1ULL);
+	info->txp->outputs[0].is_to_external = false;
+
+	if (json_to_address_scriptpubkey(info->txp, chainparams, buf, addr,
+					 &info->txp->outputs[0].script)
+	    != ADDRESS_PARSE_SUCCESS) {
+		return command_fail(cmd, LIGHTNINGD,
+				    "Change address '%.*s' unparsable?",
+				    addr->end - addr->start,
+				    buf + addr->start);
+	}
+
+	info->txp->weight = bitcoin_tx_core_weight(0, 1)
+		+ bitcoin_tx_output_weight(tal_bytelen(info->txp->outputs[0].script));
+
+	/* Find all the utxos we want to spend on this tx */
+	req = jsonrpc_request_start(cmd->plugin, cmd,
+				    "listfunds",
+				    listfunds_done,
+				    forward_error,
+				    info);
+	return send_outreq(cmd->plugin, req);
+}
+
+static struct command_result *json_upgradewallet(struct command *cmd,
+						 const char *buffer,
+						 const jsmntok_t *params)
+{
+	bool *reservedok;
+	struct out_req *req;
+	struct listfunds_info *info = tal(cmd, struct listfunds_info);
+
+	if (!param(cmd, buffer, params,
+		   p_opt("feerate", param_string, &info->feerate),
+		   p_opt_def("reservedok", param_bool, &reservedok, false),
+		   NULL))
+		return command_param_failed();
+
+	info->reservedok = *reservedok;
+	/* Get an address to send everything to */
+	req = jsonrpc_request_start(cmd->plugin, cmd,
+				    "newaddr",
+				    newaddr_sweep_done,
+				    forward_error,
+				    info);
+	return send_outreq(cmd->plugin, req);
 }
 
 static const struct plugin_command commands[] = {
@@ -565,6 +720,13 @@ static const struct plugin_command commands[] = {
 		"Send to {destination} {satoshi} (or 'all') at optional {feerate} using utxos from {minconf} or {utxos}.",
 		json_withdraw
 	},
+	{
+		"upgradewallet",
+		"bitcoin",
+		"Spend p2sh wrapped outputs into a native segwit output",
+		"Send all p2sh-wrapped outputs to a bech32 native segwit address",
+		json_upgradewallet
+	},
 };
 
 #if DEVELOPER

tests/test_wallet.py

@@ -3,6 +3,7 @@ from decimal import Decimal
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
 from pyln.client import RpcError, Millisatoshi
+from shutil import copyfile
 from utils import (
     only_one, wait_for, sync_blockheight, EXPERIMENTAL_FEATURES,
     VALGRIND, check_coin_moves, TailableProc, scriptpubkey_addr,
@@ -1496,3 +1497,59 @@ def test_withdraw_bech32m(node_factory, bitcoind):
     for addr in addrs:
         args += [{addr: 10**3}]
     l1.rpc.multiwithdraw(args)["txid"]
+
+
+@unittest.skipIf(TEST_NETWORK != 'regtest', "Address is network specific")
+def test_upgradewallet(node_factory, bitcoind):
+    # Make sure bitcoind doesn't think it's going backwards
+    bitcoind.generate_block(104)
+    l1 = node_factory.get_node(start=False)
+
+    # Write the data/p2sh_wallet_hsm_secret to the hsm_path,
+    # so node can spend funds at p2sh_wrapped_addr
+    p2sh_wrapped_addr = '2N2V4ee2vMkiXe5FSkRqFjQhiS9hKqNytv3'
+    hsm_path_dest = os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, "hsm_secret")
+    hsm_path_origin = os.path.join('tests/data', 'p2sh_wallet_hsm_secret')
+    copyfile(hsm_path_origin, hsm_path_dest)
+
+    l1.start()
+    assert l1.daemon.is_in_log('Server started with public key 0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518')
+
+    # No funds in wallet, upgrading does nothing
+    upgrade = l1.rpc.upgradewallet()
+    assert upgrade['upgraded_outs'] == 0
+
+    l1.fundwallet(10000000, addrtype="bech32")
+
+    # Funds are in wallet but they're already native segwit
+    upgrade = l1.rpc.upgradewallet()
+    assert upgrade['upgraded_outs'] == 0
+
+    # Send funds to wallet-compatible p2sh-segwit funds
+    txid = bitcoind.rpc.sendtoaddress(p2sh_wrapped_addr, 20000000 / 10 ** 8)
+    bitcoind.generate_block(1)
+    l1.daemon.wait_for_log('Owning output .* txid {} CONFIRMED'.format(txid))
+
+    upgrade = l1.rpc.upgradewallet()
+    assert upgrade['upgraded_outs'] == 1
+    assert bitcoind.rpc.getmempoolinfo()['size'] == 1
+
+    # Should be reserved!
+    res_funds = only_one([out for out in l1.rpc.listfunds()['outputs'] if out['reserved']])
+    assert 'redeemscript' in res_funds
+
+    # Running it again should be no-op because reservedok is false
+    upgrade = l1.rpc.upgradewallet()
+    assert upgrade['upgraded_outs'] == 0
+
+    # Doing it with 'reserved ok' should have 1
+    # We use a big feerate so we can get over the RBF hump
+    upgrade = l1.rpc.upgradewallet(feerate="max_acceptable", reservedok=True)
+    assert upgrade['upgraded_outs'] == 1
+    assert bitcoind.rpc.getmempoolinfo()['size'] == 1
+
+    # Mine it, nothing to upgrade
+    l1.bitcoin.generate_block(1)
+    sync_blockheight(l1.bitcoin, [l1])
+    upgrade = l1.rpc.upgradewallet(feerate="max_acceptable", reservedok=True)
+    assert upgrade['upgraded_outs'] == 0