Add ability to fully spend utxos (#2063)

* Let RawTxSigner dummy sign transactions

* Add ability to fully spend utxos

* Fix NeutrinoNodeWithWalletTest

* Fix test, simplify call

core-test/src/test/scala/org/bitcoins/core/wallet/builder/RawTxSignerTest.scala

@@ -14,7 +14,12 @@ import org.bitcoins.core.wallet.utxo.{
   LockTimeInputInfo,
   ScriptSignatureParams
 }
-import org.bitcoins.crypto.{DoubleSha256DigestBE, ECPrivateKey}
+import org.bitcoins.crypto.{
+  DoubleSha256DigestBE,
+  ECPrivateKey,
+  LowRDummyECDigitalSignature,
+  Sign
+}
 import org.bitcoins.testkit.Implicits._
 import org.bitcoins.testkit.core.gen.{CreditingTxGen, ScriptGenerators}
 import org.bitcoins.testkit.util.BitcoinSAsyncTest
@@ -303,6 +308,57 @@ class RawTxSignerTest extends BitcoinSAsyncTest {
     }
   }
 
+  it should "dummy sign a mix of spks in a tx and then have fail verification" in {
+    forAllAsync(CreditingTxGen.inputsAndOutputs(),
+                ScriptGenerators.scriptPubKey) {
+      case ((creditingTxsInfo, destinations), (changeSPK, _)) =>
+        val fee = SatoshisPerVirtualByte(Satoshis(1000))
+
+        val dummySpendingInfos = creditingTxsInfo.map { spendingInfo =>
+          val inputInfo = spendingInfo.inputInfo
+          val mockSigners =
+            inputInfo.pubKeys.take(inputInfo.requiredSigs).map(Sign.dummySign)
+
+          inputInfo.toSpendingInfo(EmptyTransaction,
+                                   mockSigners,
+                                   HashType.sigHashAll)
+        }
+
+        val utxF =
+          StandardNonInteractiveFinalizer.txFrom(outputs = destinations,
+                                                 utxos = dummySpendingInfos,
+                                                 feeRate = fee,
+                                                 changeSPK = changeSPK)
+        val txF = utxF.flatMap(utx =>
+          RawTxSigner.sign(utx,
+                           dummySpendingInfos.toVector,
+                           RawTxSigner.emptyInvariant,
+                           dummySign = true))
+
+        // Can't use BitcoinScriptUtil.verifyScript because it will pass for things
+        // with EmptyScriptPubKeys or Multisig with 0 required sigs
+        txF.map {
+          case EmptyTransaction =>
+            succeed
+          case btx: BaseTransaction =>
+            assert(
+              btx.inputs.forall(_.scriptSignature.signatures.forall(
+                _ == LowRDummyECDigitalSignature)))
+          case wtx: WitnessTransaction =>
+            assert(
+              wtx.witness.witnesses.forall {
+                case p2wsh: P2WSHWitnessV0 =>
+                  p2wsh.signatures.forall(_ == LowRDummyECDigitalSignature)
+                case p2wpkh: P2WPKHWitnessV0 =>
+                  p2wpkh.signature == LowRDummyECDigitalSignature
+                case EmptyScriptWitness =>
+                  true
+              }
+            )
+        }
+    }
+  }
+
   it should "sign a mix of p2sh/p2wsh in a tx and then have it verified" in {
     forAllAsync(CreditingTxGen.inputsAndOutputs(CreditingTxGen.nestedOutputs),
                 ScriptGenerators.scriptPubKey) {

core/src/main/scala/org/bitcoins/core/api/wallet/WalletApi.scala

@@ -261,6 +261,32 @@ trait WalletApi extends StartStopAsync[WalletApi] {
     } yield tx
   }
 
+  def sendFromOutPoints(
+      outPoints: Vector[TransactionOutPoint],
+      address: BitcoinAddress,
+      feeRate: FeeUnit)(implicit ec: ExecutionContext): Future[Transaction]
+
+  def sendFromOutPoints(
+      outPoints: Vector[TransactionOutPoint],
+      address: BitcoinAddress,
+      feeRateOpt: Option[FeeUnit]
+  )(implicit ec: ExecutionContext): Future[Transaction] = {
+    for {
+      feeRate <- determineFeeRate(feeRateOpt)
+      tx <- sendFromOutPoints(outPoints, address, feeRate)
+    } yield tx
+  }
+
+  def emptyWallet(address: BitcoinAddress, feeRateOpt: Option[FeeUnit])(implicit
+      ec: ExecutionContext): Future[Transaction] = {
+    for {
+      feeRate <- determineFeeRate(feeRateOpt)
+      utxos <- listUtxos()
+      outPoints = utxos.map(_.outPoint)
+      tx <- sendFromOutPoints(outPoints, address, feeRate)
+    } yield tx
+  }
+
   def sendWithAlgo(
       address: BitcoinAddress,
       amount: CurrencyUnit,

core/src/main/scala/org/bitcoins/core/protocol/transaction/TxUtil.scala

@@ -7,7 +7,12 @@ import org.bitcoins.core.protocol.script._
 import org.bitcoins.core.script.control.OP_RETURN
 import org.bitcoins.core.script.crypto.HashType
 import org.bitcoins.core.util.BitcoinSLogger
-import org.bitcoins.core.wallet.builder.TxBuilderError
+import org.bitcoins.core.wallet.builder.{
+  AddWitnessDataFinalizer,
+  RawTxBuilder,
+  RawTxSigner,
+  TxBuilderError
+}
 import org.bitcoins.core.wallet.fee.FeeUnit
 import org.bitcoins.core.wallet.signer.BitcoinSigner
 import org.bitcoins.core.wallet.utxo._
@@ -124,6 +129,37 @@ object TxUtil extends BitcoinSLogger {
     calcLockTimeForInfos(utxos.map(_.inputInfo))
   }
 
+  def buildDummyTx(
+      utxos: Vector[InputInfo],
+      outputs: Vector[TransactionOutput])(implicit
+      ec: ExecutionContext): Future[Transaction] = {
+    val dummySpendingInfos = utxos.map { inputInfo =>
+      val mockSigners =
+        inputInfo.pubKeys.take(inputInfo.requiredSigs).map(Sign.dummySign)
+
+      inputInfo.toSpendingInfo(EmptyTransaction,
+                               mockSigners,
+                               HashType.sigHashAll)
+    }
+
+    val dummyInputs = utxos.map { inputInfo =>
+      TransactionInput(inputInfo.outPoint,
+                       EmptyScriptSignature,
+                       Policy.sequence)
+    }
+
+    val txBuilder = RawTxBuilder() ++= dummyInputs ++= outputs
+    val withFinalizer = txBuilder.setFinalizer(AddWitnessDataFinalizer(utxos))
+
+    for {
+      utx <- withFinalizer.buildTx()
+      signed <- RawTxSigner.sign(utx,
+                                 dummySpendingInfos,
+                                 RawTxSigner.emptyInvariant,
+                                 dummySign = true)
+    } yield signed
+  }
+
   /** Inserts script signatures and (potentially) witness data to a given
     * transaction using DummyECDigitalSignatures for all sigs in order
     * to produce a transaction roughly the size of the expected fully signed

core/src/main/scala/org/bitcoins/core/wallet/builder/RawTxSigner.scala

@@ -2,13 +2,7 @@ package org.bitcoins.core.wallet.builder
 
 import org.bitcoins.core.crypto.TxSigComponent
 import org.bitcoins.core.protocol.script.ScriptWitness
-import org.bitcoins.core.protocol.transaction.{
+import org.bitcoins.core.protocol.transaction._
-  EmptyWitness,
-  Transaction,
-  TransactionWitness,
-  TxUtil,
-  WitnessTransaction
-}
 import org.bitcoins.core.util.BitcoinSLogger
 import org.bitcoins.core.wallet.fee.FeeUnit
 import org.bitcoins.core.wallet.signer.BitcoinSigner
@@ -19,7 +13,7 @@ import org.bitcoins.core.wallet.utxo.{
 }
 
 import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success, Try}
+import scala.util.{Success, Try}
 
 /** Transactions that have been finalized by a RawTxFinalizer are passed as inputs
   * to a sign function here in order to generate fully signed transactions.
@@ -29,6 +23,40 @@ import scala.util.{Failure, Success, Try}
   */
 object RawTxSigner extends BitcoinSLogger {
 
+  val emptyInvariant: (
+      Vector[ScriptSignatureParams[InputInfo]],
+      Transaction) => Boolean = (_, _) => true
+
+  def feeInvariant(expectedFeeRate: FeeUnit): (
+      Vector[ScriptSignatureParams[InputInfo]],
+      Transaction) => Boolean =
+    addFeeRateInvariant(expectedFeeRate, emptyInvariant)
+
+  private def addFeeRateInvariant(
+      expectedFeeRate: FeeUnit,
+      userInvariants: (
+          Vector[ScriptSignatureParams[InputInfo]],
+          Transaction) => Boolean): (
+      Vector[ScriptSignatureParams[InputInfo]],
+      Transaction) => Boolean = { (utxoInfos, signedTx) =>
+    {
+      userInvariants(utxoInfos, signedTx) &&
+      TxUtil
+        .sanityChecks(isSigned = true,
+                      inputInfos = utxoInfos.map(_.inputInfo),
+                      expectedFeeRate = expectedFeeRate,
+                      tx = signedTx)
+        .isSuccess
+    }
+  }
+
+  def sign(
+      utx: Transaction,
+      utxoInfos: Vector[ScriptSignatureParams[InputInfo]])(implicit
+      ec: ExecutionContext): Future[Transaction] = {
+    sign(utx, utxoInfos, emptyInvariant, dummySign = false)
+  }
+
   def sign(txWithInfo: FinalizedTxWithSigningInfo, expectedFeeRate: FeeUnit)(
       implicit ec: ExecutionContext): Future[Transaction] = {
     sign(txWithInfo.finalizedTx, txWithInfo.infos, expectedFeeRate)
@@ -39,29 +67,53 @@ object RawTxSigner extends BitcoinSLogger {
       utxoInfos: Vector[ScriptSignatureParams[InputInfo]],
       expectedFeeRate: FeeUnit)(implicit
       ec: ExecutionContext): Future[Transaction] = {
-    sign(utx, utxoInfos, expectedFeeRate, (_, _) => true)
-  }
 
-  def sign(
+    val invariants = feeInvariant(expectedFeeRate)
-      txWithInfo: FinalizedTxWithSigningInfo,
+
-      expectedFeeRate: FeeUnit,
+    sign(utx, utxoInfos, invariants, dummySign = false)
-      invariants: (
-          Vector[ScriptSignatureParams[InputInfo]],
-          Transaction) => Boolean)(implicit
-      ec: ExecutionContext): Future[Transaction] = {
-    sign(txWithInfo.finalizedTx, txWithInfo.infos, expectedFeeRate, invariants)
   }
 
   def sign(
       utx: Transaction,
       utxoInfos: Vector[ScriptSignatureParams[InputInfo]],
       expectedFeeRate: FeeUnit,
-      invariants: (
+      userInvariants: (
           Vector[ScriptSignatureParams[InputInfo]],
           Transaction) => Boolean)(implicit
       ec: ExecutionContext): Future[Transaction] = {
-    require(utxoInfos.length == utx.inputs.length,
+
-            "Must provide exactly one UTXOSatisfyingInfo per input.")
+    val invariants = addFeeRateInvariant(expectedFeeRate, userInvariants)
+
+    sign(utx, utxoInfos, invariants, dummySign = false)
+  }
+
+  def sign(
+      txWithInfo: FinalizedTxWithSigningInfo,
+      expectedFeeRate: FeeUnit,
+      userInvariants: (
+          Vector[ScriptSignatureParams[InputInfo]],
+          Transaction) => Boolean)(implicit
+      ec: ExecutionContext): Future[Transaction] = {
+
+    val invariants = addFeeRateInvariant(expectedFeeRate, userInvariants)
+
+    sign(txWithInfo.finalizedTx,
+         txWithInfo.infos,
+         invariants,
+         dummySign = false)
+  }
+
+  def sign(
+      utx: Transaction,
+      utxoInfos: Vector[ScriptSignatureParams[InputInfo]],
+      invariants: (
+          Vector[ScriptSignatureParams[InputInfo]],
+          Transaction) => Boolean,
+      dummySign: Boolean)(implicit
+      ec: ExecutionContext): Future[Transaction] = {
+    require(
+      utxoInfos.length == utx.inputs.length,
+      s"Must provide exactly one UTXOSatisfyingInfo per input, ${utxoInfos.length} != ${utx.inputs.length}")
     require(utxoInfos.distinct.length == utxoInfos.length,
             "All UTXOSatisfyingInfos must be unique. ")
     require(utxoInfos.forall(utxo =>
@@ -81,7 +133,7 @@ object RawTxSigner extends BitcoinSLogger {
 
         val inputAndWitnessFs = utxoInfos.map { utxo =>
           val txSigCompF =
-            BitcoinSigner.sign(utxo, utx, isDummySignature = false)
+            BitcoinSigner.sign(utxo, utx, isDummySignature = dummySign)
           txSigCompF.map { txSigComp =>
             val scriptWitnessOpt = TxSigComponent.getScriptWitness(txSigComp)
 
@@ -125,14 +177,7 @@ object RawTxSigner extends BitcoinSLogger {
     signedTxF.flatMap { signedTx =>
       val txT: Try[Transaction] = {
         if (invariants(utxoInfos, signedTx)) {
-          //final sanity checks
+          Success(signedTx)
-          TxUtil.sanityChecks(isSigned = true,
-                              inputInfos = utxoInfos.map(_.inputInfo),
-                              expectedFeeRate = expectedFeeRate,
-                              tx = signedTx) match {
-            case Success(_)   => Success(signedTx)
-            case Failure(err) => Failure(err)
-          }
         } else {
           TxBuilderError.FailedUserInvariants
         }

core/src/main/scala/org/bitcoins/core/wallet/signer/Signer.scala

@@ -11,12 +11,7 @@ import org.bitcoins.core.script.crypto.HashType
 import org.bitcoins.core.script.flag.ScriptFlag
 import org.bitcoins.core.wallet.builder.TxBuilderError
 import org.bitcoins.core.wallet.utxo._
-import org.bitcoins.crypto.{
+import org.bitcoins.crypto._
-  DummyECDigitalSignature,
-  ECDigitalSignature,
-  ECPublicKey,
-  Sign
-}
 import scodec.bits.ByteVector
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -45,7 +40,7 @@ sealed abstract class SignerUtils {
       isDummySignature: Boolean)(implicit
       ec: ExecutionContext): Future[ECDigitalSignature] = {
     if (isDummySignature) {
-      Future.successful(DummyECDigitalSignature)
+      Future.successful(LowRDummyECDigitalSignature)
     } else {
       TransactionSignatureCreator.createSig(unsignedTx,
                                             signingInfo,

crypto/src/main/scala/org/bitcoins/crypto/ECDigitalSignature.scala

@@ -103,6 +103,17 @@ case object DummyECDigitalSignature extends ECDigitalSignature {
   override def s: BigInt = r
 }
 
+/**
+  * The point of this case object is to help with fee estimation
+  * when using low r signing. Technically this number can vary,
+  * 71 bytes is the most likely when using low r signing
+  */
+case object LowRDummyECDigitalSignature extends ECDigitalSignature {
+  override val bytes: ByteVector = ByteVector(Array.fill(71)(0.toByte))
+  override def r: BigInt = EmptyDigitalSignature.r
+  override def s: BigInt = r
+}
+
 object ECDigitalSignature extends Factory[ECDigitalSignature] {
 
   private case class ECDigitalSignatureImpl(bytes: ByteVector)
@@ -111,7 +122,12 @@ object ECDigitalSignature extends Factory[ECDigitalSignature] {
   override def fromBytes(bytes: ByteVector): ECDigitalSignature = {
     //this represents the empty signature
     if (bytes.size == 1 && bytes.head == 0x0) EmptyDigitalSignature
-    else if (bytes.size == 0) EmptyDigitalSignature
+    else if (bytes.size == 0)
+      EmptyDigitalSignature
+    else if (bytes == DummyECDigitalSignature.bytes)
+      DummyECDigitalSignature
+    else if (bytes == LowRDummyECDigitalSignature.bytes)
+      LowRDummyECDigitalSignature
     else {
       //make sure the signature follows BIP62's low-s value
       //https://github.com/bitcoin/bips/blob/master/bip-0062.mediawiki#Low_S_values_in_signatures

node-test/src/test/scala/org/bitcoins/node/NeutrinoNodeWithWalletTest.scala

@@ -60,7 +60,7 @@ class NeutrinoNodeWithWalletTest extends NodeUnitTest {
 
   val TestAmount = 1.bitcoin
   val FeeRate = SatoshisPerByte(10.sats)
-  val TestFees = 2240.sats
+  val TestFees: Satoshis = 2230.sats
 
   def nodeCallbacks: NodeCallbacks = {
     val onBlock: OnBlockReceived = { block =>
@@ -99,8 +99,9 @@ class NeutrinoNodeWithWalletTest extends NodeUnitTest {
           addresses <- wallet.listAddresses()
           utxos <- wallet.listDefaultAccountUtxos()
         } yield {
-          (expectedConfirmedAmount == confirmedBalance) &&
+          // +- fee rate because signatures could vary in size
-          (expectedUnconfirmedAmount == unconfirmedBalance) &&
+          (expectedConfirmedAmount === confirmedBalance +- FeeRate.currencyUnit) &&
+          (expectedUnconfirmedAmount === unconfirmedBalance +- FeeRate.currencyUnit) &&
           (expectedAddresses == addresses.size) &&
           (expectedUtxos == utxos.size)
         }

wallet-test/src/test/scala/org/bitcoins/wallet/WalletSendingTest.scala

@@ -6,11 +6,12 @@ import org.bitcoins.core.protocol.BitcoinAddress
 import org.bitcoins.core.protocol.transaction.TransactionOutput
 import org.bitcoins.core.script.constant.{BytesToPushOntoStack, ScriptConstant}
 import org.bitcoins.core.script.control.OP_RETURN
-import org.bitcoins.core.wallet.fee.SatoshisPerByte
+import org.bitcoins.core.wallet.fee._
 import org.bitcoins.core.wallet.utxo.TxoState
 import org.bitcoins.crypto.CryptoUtil
 import org.bitcoins.testkit.core.gen.{CurrencyUnitGenerator, FeeUnitGen}
 import org.bitcoins.testkit.wallet.BitcoinSWalletTest
+import org.bitcoins.testkit.wallet.BitcoinSWalletTest.RandomFeeProvider
 import org.bitcoins.testkit.wallet.FundWalletUtil.FundedWallet
 import org.scalatest.{Assertion, FutureOutcome}
 import scodec.bits.ByteVector
@@ -210,6 +211,51 @@ class WalletSendingTest extends BitcoinSWalletTest {
     }
   }
 
+  it should "correctly send entire outpoints" in { fundedWallet =>
+    val wallet = fundedWallet.wallet
+    for {
+      allUtxos <- wallet.listUtxos()
+      // use half of them
+      utxos = allUtxos.drop(allUtxos.size / 2)
+      outPoints = utxos.map(_.outPoint)
+      tx <- wallet.sendFromOutPoints(outPoints, testAddress, None)
+    } yield {
+      val expectedFeeRate =
+        wallet.feeRateApi.asInstanceOf[RandomFeeProvider].lastFeeRate.get
+      assert(outPoints.forall(outPoint =>
+               tx.inputs.exists(_.previousOutput == outPoint)),
+             "Every outpoint was not included included")
+      assert(tx.inputs.size == outPoints.size, "An extra input was added")
+
+      val inputAmount = utxos.map(_.output.value).sum
+      val numInputs = outPoints.size
+
+      val defaultRange = expectedFeeRate.scaleFactor * numInputs
+
+      val (actualFeeRate, range) = expectedFeeRate match {
+        case _: SatoshisPerByte =>
+          (SatoshisPerByte.calc(inputAmount, tx), defaultRange)
+        case _: SatoshisPerVirtualByte =>
+          (SatoshisPerVirtualByte.calc(inputAmount, tx), defaultRange)
+        case _: SatoshisPerKiloByte =>
+          (SatoshisPerKiloByte.calc(inputAmount, tx), defaultRange)
+        case _: SatoshisPerKW =>
+          // multiply range by 4 because an extra byte on a sig counts as 4 weight units
+          (SatoshisPerKW.calc(inputAmount, tx), defaultRange * 4)
+      }
+
+      // +- range in case of rounding or unexpected signature sizes
+      assert(
+        actualFeeRate.toLong === expectedFeeRate.toLong +- range,
+        s"Expected fee rate: $expectedFeeRate, inputs: $numInputs input: $inputAmount, tx bytes: ${tx.byteSize} " +
+          s"vsize: ${tx.vsize} weight ${tx.weight}"
+      )
+
+      assert(tx.outputs.size == 1)
+      assert(tx.outputs.head.scriptPubKey == testAddress.scriptPubKey)
+    }
+  }
+
   it should "fail to send from outpoints when already spent" in {
     fundedWallet =>
       val wallet = fundedWallet.wallet

wallet/src/main/scala/org/bitcoins/wallet/Wallet.scala

@@ -13,9 +13,10 @@ import org.bitcoins.core.crypto.ExtPublicKey
 import org.bitcoins.core.currency._
 import org.bitcoins.core.gcs.{GolombFilter, SimpleFilterMatcher}
 import org.bitcoins.core.hd.{HDAccount, HDCoin, HDPurpose, HDPurposes}
+import org.bitcoins.core.policy.Policy
 import org.bitcoins.core.protocol.BitcoinAddress
 import org.bitcoins.core.protocol.blockchain.ChainParams
-import org.bitcoins.core.protocol.script.ScriptPubKey
+import org.bitcoins.core.protocol.script.{EmptyScriptPubKey, ScriptPubKey}
 import org.bitcoins.core.protocol.transaction._
 import org.bitcoins.core.script.constant.ScriptConstant
 import org.bitcoins.core.script.control.OP_RETURN
@@ -25,7 +26,7 @@ import org.bitcoins.core.wallet.builder.{
   RawTxSigner,
   ShufflingNonInteractiveFinalizer
 }
-import org.bitcoins.core.wallet.fee.FeeUnit
+import org.bitcoins.core.wallet.fee._
 import org.bitcoins.core.wallet.keymanagement.{
   KeyManagerParams,
   KeyManagerUnlockError
@@ -389,6 +390,57 @@ abstract class Wallet
     }
   }
 
+  def sendFromOutPoints(
+      outPoints: Vector[TransactionOutPoint],
+      address: BitcoinAddress,
+      feeRate: FeeUnit)(implicit ec: ExecutionContext): Future[Transaction] = {
+    require(
+      address.networkParameters.isSameNetworkBytes(networkParameters),
+      s"Cannot send to address on other network, got ${address.networkParameters}"
+    )
+    logger.info(s"Sending to $address at feerate $feeRate")
+    for {
+      utxoDbs <- spendingInfoDAO.findByOutPoints(outPoints)
+      diff = utxoDbs.map(_.outPoint).diff(outPoints)
+      _ = require(diff.isEmpty,
+                  s"Not all OutPoints belong to this wallet, diff $diff")
+      spentUtxos =
+        utxoDbs.filterNot(utxo => TxoState.receivedStates.contains(utxo.state))
+      _ = require(
+        spentUtxos.isEmpty,
+        s"Some out points given have already been spent, ${spentUtxos.map(_.outPoint)}")
+
+      utxos <- Future.sequence {
+        utxoDbs.map(utxo =>
+          transactionDAO
+            .findByOutPoint(utxo.outPoint)
+            .map(txDb => utxo.toUTXOInfo(keyManager, txDb.get.transaction)))
+      }
+
+      utxoAmount = utxoDbs.map(_.output.value).sum
+      dummyOutput = TransactionOutput(utxoAmount, address.scriptPubKey)
+
+      dummyTx <-
+        TxUtil.buildDummyTx(utxos.map(_.inputInfo), Vector(dummyOutput))
+
+      fee = feeRate * dummyTx
+      amount = utxoAmount - fee
+
+      _ = require(amount > Policy.dustThreshold,
+                  "Utxos are not large enough to send at this fee rate")
+
+      output = TransactionOutput(amount, address.scriptPubKey)
+      txBuilder = ShufflingNonInteractiveFinalizer.txBuilderFrom(
+        Vector(output),
+        utxos,
+        feeRate,
+        EmptyScriptPubKey // There will be no change
+      )
+
+      tx <- finishSend(txBuilder, utxos, amount, feeRate, Vector.empty)
+    } yield tx
+  }
+
   override def sendFromOutPoints(
       outPoints: Vector[TransactionOutPoint],
       address: BitcoinAddress,