1
0
mirror of https://github.com/ACINQ/eclair.git synced 2024-11-20 10:39:19 +01:00

Electrum: Use actual tx weight when estimating fees (#321)

* Core fee estimator: return result in satoshi per byte
It was still in satoshi per kb

* Electrum: use actual tx weight to estimate fees when funding a tx

This fixes #303
This commit is contained in:
Fabrice Drouin 2017-12-22 20:15:46 +01:00 committed by Pierre-Marie Padiou
parent dda2d94702
commit 3051cc5602
5 changed files with 204 additions and 70 deletions

View File

@ -10,6 +10,7 @@ import fr.acinq.bitcoin.{Base58, Base58Check, BinaryData, Block, Crypto, Determi
import fr.acinq.eclair.blockchain.bitcoind.rpc.Error
import fr.acinq.eclair.blockchain.electrum.ElectrumClient.{GetTransaction, GetTransactionResponse, TransactionHistoryItem, computeScriptHash}
import fr.acinq.eclair.randomBytes
import fr.acinq.eclair.transactions.Transactions
import grizzled.slf4j.Logging
import scala.annotation.tailrec
@ -408,18 +409,18 @@ object ElectrumWallet {
* - 0 means unconfirmed, but all input are confirmed
* < 0 means unconfirmed, and sonme inputs are unconfirmed as well
*
* @param tip current blockchain tip
* @param accountKeys account keys
* @param changeKeys change keys
* @param status script hash -> status; "" means that the script hash has not been used
* yet
* @param transactions wallet transactions
* @param heights transactions heights
* @param history script hash -> history
* @param locks transactions which lock some of our utxos.
* @param pendingHistoryRequests requests pending a response from the electrum server
* @param pendingTransactionRequests requests pending a response from the electrum server
* @param pendingTransactions transactions received but not yet connected to their parents
* @param tip current blockchain tip
* @param accountKeys account keys
* @param changeKeys change keys
* @param status script hash -> status; "" means that the script hash has not been used
* yet
* @param transactions wallet transactions
* @param heights transactions heights
* @param history script hash -> history
* @param locks transactions which lock some of our utxos.
* @param pendingHistoryRequests requests pending a response from the electrum server
* @param pendingTransactionRequests requests pending a response from the electrum server
* @param pendingTransactions transactions received but not yet connected to their parents
*/
case class Data(tip: ElectrumClient.Header,
accountKeys: Vector[ExtendedPrivateKey],
@ -608,6 +609,48 @@ object ElectrumWallet {
}
}
/**
*
* @param tx input transaction
* @param utxos input uxtos
* @return a tx where all utxos have been added as inputs, signed with dummy invalid signatures. This
* is used to estimate the weight of the signed transaction
*/
def addUtxosWithDummySig(tx: Transaction, utxos: Seq[Utxo]): Transaction =
tx.copy(txIn = utxos.map { case utxo =>
// we use dummy signature here, because the result is only used to estimate fees
val sig = BinaryData("01" * 71)
val sigScript = Script.write(OP_PUSHDATA(Script.write(Script.pay2wpkh(utxo.key.publicKey))) :: Nil)
val witness = ScriptWitness(sig :: utxo.key.publicKey.toBin :: Nil)
TxIn(utxo.outPoint, signatureScript = sigScript, sequence = TxIn.SEQUENCE_FINAL, witness = witness)
})
/**
*
* @param amount amount we want to pay
* @param allowSpendUnconfirmed if true, use unconfirmed utxos
* @return a set of utxos with a total value that is greater than amount
*/
def chooseUtxos(amount: Satoshi, allowSpendUnconfirmed: Boolean): Seq[Utxo] = {
@tailrec
def select(chooseFrom: Seq[Utxo], selected: Set[Utxo]): Set[Utxo] = {
if (totalAmount(selected) >= amount) selected
else if (chooseFrom.isEmpty) throw new IllegalArgumentException("insufficient funds")
else select(chooseFrom.tail, selected + chooseFrom.head)
}
// select utxos that are not locked by pending txs
val lockedOutputs = locks.map(_.txIn.map(_.outPoint)).flatten
val unlocked = utxos.filterNot(utxo => lockedOutputs.contains(utxo.outPoint))
val unlocked1 = if (allowSpendUnconfirmed) unlocked else unlocked.filter(_.item.height > 0)
// sort utxos by amount, in increasing order
// this way we minimize the number of utxos in the wallet, and so we minimize the fees we'll pay for them
val unlocked2 = unlocked1.sortBy(_.item.value)
val selected = select(unlocked2, Set())
selected.toSeq
}
/**
*
* @param tx input tx that has no inputs
@ -624,42 +667,41 @@ object ElectrumWallet {
require(feeRatePerKw >= 0, "fee rate cannot be negative")
val amount = tx.txOut.map(_.amount).sum
require(amount > dustLimit, "amount to send is below dust limit")
val fee = {
val estimatedFee = computeFee(700, feeRatePerKw)
if (estimatedFee < minimumFee) minimumFee else estimatedFee
// start with a hefty fee estimate
val utxos = chooseUtxos(amount + Transactions.weight2fee(feeRatePerKw, 1000), allowSpendUnconfirmed)
val spent = totalAmount(utxos)
// add utxos, and sign with dummy sigs
val tx1 = addUtxosWithDummySig(tx, utxos)
// compute the actual fee that we should pay
val fee1 = {
// add a dummy change output, which will be needed most of the time
val tx2 = tx1.addOutput(TxOut(amount, computePublicKeyScript(currentChangeKey.publicKey)))
Transactions.weight2fee(feeRatePerKw, tx2.weight())
}
@tailrec
def select(chooseFrom: Seq[Utxo], selected: Set[Utxo]): Set[Utxo] = {
if (totalAmount(selected) >= amount + fee) selected
else if (chooseFrom.isEmpty) Set()
else select(chooseFrom.tail, selected + chooseFrom.head)
// add change output only if non-dust, otherwise change is added to the fee
val (tx2, fee2, pos) = (spent - amount - fee1) match {
case dustChange if dustChange < dustLimit => (tx1, fee1 + dustChange, -1) // if change is below dust we add it to fees
case change => (tx1.addOutput(TxOut(change, computePublicKeyScript(currentChangeKey.publicKey))), fee1, 1) // change output index is always 1
}
// select utxos that are not locked by pending txs
val lockedOutputs = locks.map(_.txIn.map(_.outPoint)).flatten
val unlocked = utxos.filterNot(utxo => lockedOutputs.contains(utxo.outPoint))
val unlocked1 = if (allowSpendUnconfirmed) unlocked else unlocked.filter(_.item.height > 0)
val selected = select(unlocked1, Set()).toSeq
require(totalAmount(selected) >= amount + fee, "insufficient funds")
// sign our tx
val tx3 = tx2.copy(txIn = tx2.txIn.zipWithIndex.map { case (txIn, i) =>
val key = utxos(i).key
val sig = Transaction.signInput(tx2, i, Script.pay2pkh(key.publicKey), SIGHASH_ALL, Satoshi(utxos(i).item.value), SigVersion.SIGVERSION_WITNESS_V0, key.privateKey)
val sigScript = Script.write(OP_PUSHDATA(Script.write(Script.pay2wpkh(key.publicKey))) :: Nil)
val witness = ScriptWitness(sig :: key.publicKey.toBin :: Nil)
txIn.copy(signatureScript = sigScript, witness = witness)
})
//Transaction.correctlySpends(tx3, utxos.map(utxo => utxo.outPoint -> TxOut(Satoshi(utxo.item.value), computePublicKeyScript(utxo.key.publicKey))).toMap, ScriptFlags.STANDARD_SCRIPT_VERIFY_FLAGS)
// add inputs
var tx1 = tx.copy(txIn = selected.map(utxo => TxIn(utxo.outPoint, Nil, TxIn.SEQUENCE_FINAL)))
// and add the completed tx to the lokcs
val data1 = this.copy(locks = this.locks + tx3)
// add change output
val change = totalAmount(selected) - amount - fee
if (change >= dustLimit) tx1 = tx1.addOutput(TxOut(change, computePublicKeyScript(currentChangeKey.publicKey)))
// sign
for (i <- 0 until tx1.txIn.size) {
val key = selected(i).key
val sig = Transaction.signInput(tx1, i, Script.pay2pkh(key.publicKey), SIGHASH_ALL, Satoshi(selected(i).item.value), SigVersion.SIGVERSION_WITNESS_V0, key.privateKey)
tx1 = tx1.updateWitness(i, ScriptWitness(sig :: key.publicKey.toBin :: Nil)).updateSigScript(i, OP_PUSHDATA(Script.write(Script.pay2wpkh(key.publicKey))) :: Nil)
}
Transaction.correctlySpends(tx1, selected.map(utxo => utxo.outPoint -> TxOut(Satoshi(utxo.item.value), computePublicKeyScript(utxo.key.publicKey))).toMap, ScriptFlags.STANDARD_SCRIPT_VERIFY_FLAGS)
val data1 = this.copy(locks = this.locks + tx1)
(data1, tx1)
(data1, tx3)
}
/**
@ -700,4 +742,6 @@ object ElectrumWallet {
= Data(tip, accountKeys, changeKeys, Map(), Map(), Map(), Map(), Set(), Set(), Set(), Seq())
}
case class InfiniteLoopException(data: Data, tx: Transaction) extends Exception
}

View File

@ -15,20 +15,20 @@ class BitcoinCoreFeeProvider(rpcClient: BitcoinJsonRPCClient, defaultFeerates: F
* We need this to keep commitment tx fees in sync with the state of the network
*
* @param nBlocks number of blocks until tx is confirmed
* @return the current fee estimate in Satoshi/Kb
* @return the current fee estimate in Satoshi/Byte
*/
def estimateSmartFee(nBlocks: Int): Future[Long] =
rpcClient.invoke("estimatesmartfee", nBlocks).map(json => {
json \ "feerate" match {
case JDouble(feerate) =>
// estimatesmartfee returns a fee rate in Btc/Kb
btc2satoshi(Btc(feerate)).amount
btc2satoshi(Btc(feerate)).amount / 1024
case JInt(feerate) if feerate.toLong < 0 =>
// negative value means failure
feerate.toLong
case JInt(feerate) =>
// should (hopefully) never happen
btc2satoshi(Btc(feerate.toLong)).amount
btc2satoshi(Btc(feerate.toLong)).amount / 1024
}
})

View File

@ -35,6 +35,7 @@ object BitgoFeeProvider {
def parseFeeRanges(json: JValue): Seq[BlockTarget] = {
val blockTargets = json \ "feeByBlockTarget"
blockTargets.foldField(Seq.empty[BlockTarget]) {
// we divide by 1024 because bitgo returns estimates in Satoshi/Kb and we use estimates in Satoshi/Byte
case (list, (strBlockTarget, JInt(feePerKb))) => list :+ BlockTarget(strBlockTarget.toInt, feePerKb.longValue() / 1024)
}
}

View File

@ -73,6 +73,14 @@ object Transactions {
def weight2fee(feeratePerKw: Long, weight: Int) = Satoshi((feeratePerKw * weight) / 1000)
/**
*
* @param fee tx fee
* @param weight tx weight
* @return the fee rate (in Satoshi/Kw) for this tx
*/
def fee2rate(fee: Satoshi, weight: Int) = (fee.amount * 1000L) / weight
def trimOfferedHtlcs(dustLimit: Satoshi, spec: CommitmentSpec): Seq[DirectedHtlc] = {
val htlcTimeoutFee = weight2fee(spec.feeratePerKw, htlcTimeoutWeight)
spec.htlcs

View File

@ -1,16 +1,20 @@
package fr.acinq.eclair.blockchain.electrum
import fr.acinq.bitcoin.Crypto.PrivateKey
import fr.acinq.bitcoin.DeterministicWallet.derivePrivateKey
import fr.acinq.bitcoin.DeterministicWallet.{ExtendedPrivateKey, derivePrivateKey}
import fr.acinq.bitcoin._
import fr.acinq.eclair.transactions.Transactions
import org.junit.runner.RunWith
import org.scalatest.FunSuite
import org.scalatest.junit.JUnitRunner
import scala.util.{Failure, Random, Success, Try}
@RunWith(classOf[JUnitRunner])
class ElectrumWalletBasicSpec extends FunSuite {
import ElectrumWallet._
import ElectrumWalletBasicSpec._
val swipeRange = 10
val dustLimit = 546 satoshi
@ -30,11 +34,30 @@ class ElectrumWalletBasicSpec extends FunSuite {
val params = ElectrumWallet.WalletParameters(Block.RegtestGenesisBlock.hash)
val state = Data(params, ElectrumClient.Header.RegtestGenesisHeader, firstAccountKeys, firstChangeKeys)
val unspents = Map(
computeScriptHashFromPublicKey(state.accountKeys(0).publicKey) -> Set(ElectrumClient.UnspentItem("01" * 32, 0, 1 * Satoshi(Coin).toLong, 100)),
computeScriptHashFromPublicKey(state.accountKeys(1).publicKey) -> Set(ElectrumClient.UnspentItem("02" * 32, 0, 2 * Satoshi(Coin).toLong, 100)),
computeScriptHashFromPublicKey(state.accountKeys(2).publicKey) -> Set(ElectrumClient.UnspentItem("03" * 32, 0, 3 * Satoshi(Coin).toLong, 100))
)
.copy(status = (firstAccountKeys ++ firstChangeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
def addFunds(data: Data, key: ExtendedPrivateKey, amount: Satoshi): Data = {
val tx = Transaction(version = 1, txIn = Nil, txOut = TxOut(amount, ElectrumWallet.computePublicKeyScript(key.publicKey)) :: Nil, lockTime = 0)
val scriptHash = ElectrumWallet.computeScriptHashFromPublicKey(key.publicKey)
val scriptHashHistory = data.history.getOrElse(scriptHash, Seq.empty[ElectrumClient.TransactionHistoryItem])
data.copy(
history = data.history.updated(scriptHash, scriptHashHistory :+ ElectrumClient.TransactionHistoryItem(100, tx.txid)),
transactions = data.transactions + (tx.txid -> tx)
)
}
def addFunds(data: Data, keyamount: (ExtendedPrivateKey, Satoshi)): Data = {
val tx = Transaction(version = 1, txIn = Nil, txOut = TxOut(keyamount._2, ElectrumWallet.computePublicKeyScript(keyamount._1.publicKey)) :: Nil, lockTime = 0)
val scriptHash = ElectrumWallet.computeScriptHashFromPublicKey(keyamount._1.publicKey)
val scriptHashHistory = data.history.getOrElse(scriptHash, Seq.empty[ElectrumClient.TransactionHistoryItem])
data.copy(
history = data.history.updated(scriptHash, scriptHashHistory :+ ElectrumClient.TransactionHistoryItem(100, tx.txid)),
transactions = data.transactions + (tx.txid -> tx)
)
}
def addFunds(data: Data, keyamounts: Seq[(ExtendedPrivateKey, Satoshi)]): Data = keyamounts.foldLeft(data)(addFunds)
test("compute addresses") {
val priv = PrivateKey.fromBase58("cRumXueoZHjhGXrZWeFoEBkeDHu2m8dW5qtFBCqSAt4LDR2Hnd8Q", Base58.Prefix.SecretKeyTestnet)
@ -52,47 +75,105 @@ class ElectrumWalletBasicSpec extends FunSuite {
assert(segwitAddress(firstKey) === "2MxJejujQJRRJdbfTKNQQ94YCnxJwRaE7yo")
}
ignore("complete transactions (enough funds)") {
val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
test("complete transactions (enough funds)") {
val state1 = addFunds(state, state.accountKeys.head, 1 btc)
val (confirmed1, unconfirmed1) = state1.balance
val pub = PrivateKey(BinaryData("01" * 32), compressed = true).publicKey
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(0.5 btc, Script.pay2pkh(pub)) :: Nil, lockTime = 0)
val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
val Some((_, _, Some(fee))) = state2.computeTransactionDelta(tx1)
val actualFeeRate = Transactions.fee2rate(fee, tx1.weight())
val state3 = state2.cancelTransaction(tx1)
assert(state3 == state1)
val state4 = state2.commitTransaction(tx1)
assert(state4.utxos.size + tx1.txIn.size == state1.utxos.size)
val (confirmed4, unconfirmed4) = state4.balance
assert(confirmed4 == confirmed1)
assert(unconfirmed1 - unconfirmed4 >= btc2satoshi(0.5 btc))
}
test("complete transactions (insufficient funds)") {
val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
val state1 = addFunds(state, state.accountKeys.head, 5 btc)
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(6 btc, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
val e = intercept[IllegalArgumentException] {
val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
}
}
ignore("find what a tx spends from us") {
val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
test("compute the effect of tx") {
val state1 = addFunds(state, state.accountKeys.head, 1 btc)
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(0.5 btc, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
val pubkeys = tx1.txIn.map(extractPubKeySpentFrom).flatten
val utxos1 = state2.utxos.filter(utxo => pubkeys.contains(utxo.key.publicKey))
val utxos2 = state2.utxos.filter(utxo => tx1.txIn.map(_.outPoint).contains(utxo.outPoint))
println(pubkeys)
val Some((received, sent, Some(fee))) = state1.computeTransactionDelta(tx1)
assert(sent - received - fee == btc2satoshi(0.5 btc))
}
ignore("find what a tx sends to us") {
val state1 = state.copy(status = (state.accountKeys ++ state.changeKeys).map(key => computeScriptHashFromPublicKey(key.publicKey) -> "").toMap)
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(0.5 btc, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
val (state2, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, false)
test("use actual transaction weight to compute fees") {
val state1 = addFunds(state, (state.accountKeys(0), Satoshi(5000000)) :: (state.accountKeys(1), Satoshi(6000000)) :: (state.accountKeys(2), Satoshi(4000000)) :: Nil)
val pubSpent = tx1.txIn.map(extractPubKeySpentFrom).flatten
val utxos1 = state2.utxos.filter(utxo => pubSpent.contains(utxo.key.publicKey))
val utxos2 = state2.utxos.filter(utxo => tx1.txIn.map(_.outPoint).contains(utxo.outPoint))
println(pubSpent)
{
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(Satoshi(5000000), Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
val (state3, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, true)
val Some((_, _, Some(fee))) = state3.computeTransactionDelta(tx1)
val actualFeeRate = Transactions.fee2rate(fee, tx1.weight())
assert(isFeerateOk(actualFeeRate, feeRatePerKw))
}
{
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(Satoshi(5000000) - dustLimit, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
val (state3, tx1) = state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, true)
val Some((_, _, Some(fee))) = state3.computeTransactionDelta(tx1)
val actualFeeRate = Transactions.fee2rate(fee, tx1.weight())
assert(isFeerateOk(actualFeeRate, feeRatePerKw))
}
{
// with a huge fee rate that will force us to use an additional input when we complete our tx
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(Satoshi(3000000), Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
val (state3, tx1) = state1.completeTransaction(tx, 100 * feeRatePerKw, minimumFee, dustLimit, true)
val Some((_, _, Some(fee))) = state3.computeTransactionDelta(tx1)
val actualFeeRate = Transactions.fee2rate(fee, tx1.weight())
assert(isFeerateOk(actualFeeRate, 100 * feeRatePerKw))
}
{
// with a tiny fee rate that will force us to use an additional input when we complete our tx
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(Btc(0.09), Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
val (state3, tx1) = state1.completeTransaction(tx, feeRatePerKw / 10, minimumFee / 10, dustLimit, true)
val Some((_, _, Some(fee))) = state3.computeTransactionDelta(tx1)
val actualFeeRate = Transactions.fee2rate(fee, tx1.weight())
assert(isFeerateOk(actualFeeRate, feeRatePerKw / 10))
}
}
test("fuzzy test") {
val random = new Random()
(0 to 10) foreach { _ =>
val funds = for (i <- 0 until random.nextInt(10)) yield {
val index = random.nextInt(state.accountKeys.length)
val amount = dustLimit + Satoshi(random.nextInt(10000000))
(state.accountKeys(index), amount)
}
val state1 = addFunds(state, funds)
(0 until 30) foreach { _ =>
val amount = dustLimit + Satoshi(random.nextInt(10000000))
val tx = Transaction(version = 2, txIn = Nil, txOut = TxOut(amount, Script.pay2pkh(state1.accountKeys(0).publicKey)) :: Nil, lockTime = 0)
Try(state1.completeTransaction(tx, feeRatePerKw, minimumFee, dustLimit, true)) match {
case Success((state2, tx1)) => ()
case Failure(cause) if cause.getMessage != null && cause.getMessage.contains("insufficient funds") => ()
case Failure(cause) => println(s"unexpected $cause")
}
}
}
}
}
object ElectrumWalletBasicSpec {
/**
*
* @param actualFeeRate actual fee rate
* @param targetFeeRate target fee rate
* @return true if actual fee rate is within 10% of target
*/
def isFeerateOk(actualFeeRate: Long, targetFeeRate: Long): Boolean = Math.abs(actualFeeRate - targetFeeRate) < 0.1 * (actualFeeRate + targetFeeRate)
}