diff --git a/docs/release-notes/eclair-vnext.md b/docs/release-notes/eclair-vnext.md index 85b01d1a39..056e002289 100644 --- a/docs/release-notes/eclair-vnext.md +++ b/docs/release-notes/eclair-vnext.md @@ -231,6 +231,11 @@ eclair.relay.peer-reputation.enabled = false eclair.relay.reserved-for-accountable = 0.0 ``` +### Faster scanning for spending transactions with Bitcoin Core's txospenderindex + +If Bitcoin Core's `txospenderindex` (available in Bitcoin Core 31.0 and newer) is available and synced, Eclair will use it to find channel spending transactions, which is much faster and less expensive than scanning blocks. +To enable this index, start Bitcoin Core with `-txospenderindex` or add `txospenderindex=1` to your `bitcoin.conf`. + ### Configuration changes diff --git a/eclair-core/pom.xml b/eclair-core/pom.xml index 10900624bf..2b265db280 100644 --- a/eclair-core/pom.xml +++ b/eclair-core/pom.xml @@ -87,8 +87,8 @@ true - https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-x86_64-linux-gnu.tar.gz - 6aa7bb4feb699c4c6262dd23e4004191f6df7f373b5d5978b5bcdd4bb72f75d8 + https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-x86_64-linux-gnu.tar.gz + d3e4c58a35b1d0a97a457462c94f55501ad167c660c245cb1ffa565641c65074 @@ -99,8 +99,8 @@ - https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-x86_64-apple-darwin.tar.gz - 99d5cee9b9c37be506396c30837a4b98e320bfea71c474d6120a7e8eb6075c7b + https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-x86_64-apple-darwin.tar.gz + 56824dd705bc2a3b22d42e8aa02ed53498d491ff7c2c8aa96831333871887ead @@ -111,8 +111,8 @@ - https://bitcoincore.org/bin/bitcoin-core-30.2/bitcoin-30.2-win64.zip - 0d7e1f16f8823aa26d29b44855ff6dbac11c03d75631a6c1d2ea5fab3a84fdf8 + https://bitcoincore.org/bin/bitcoin-core-31.0/bitcoin-31.0-win64.zip + 82fd2c504a0f20a31d4d13bd407783d6fc7bf17622d0ce85228a9b92694e03f0 diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/OnChainWallet.scala b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/OnChainWallet.scala index 37e1481a68..be5b89500f 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/OnChainWallet.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/OnChainWallet.scala @@ -81,7 +81,7 @@ trait OnChainChannelFunder { * Note that if this function returns false, that doesn't mean the output cannot be spent. The output could be unknown * (not in the blockchain nor in the mempool) but could reappear later and be spendable at that point. */ - def isTransactionOutputSpendable(txid: TxId, outputIndex: Int, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] + def isTransactionOutputSpendable(outPoint: OutPoint, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] /** Rollback a transaction that we failed to commit: this probably translates to "release locks on utxos". */ def rollback(tx: Transaction)(implicit ec: ExecutionContext): Future[Boolean] diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala index d85057e7e4..01306b2028 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala @@ -103,6 +103,8 @@ object ZmqWatcher { def txId: TxId /** Index of the outpoint to watch. */ def outputIndex: Int + /** outpoint to watch */ + def outPoint: OutPoint = OutPoint(txId, outputIndex) /** * TxIds of potential spending transactions; most of the time we know the txs, and it allows for optimizations. * This argument can safely be ignored by watcher implementations. @@ -427,6 +429,8 @@ private class ZmqWatcher(nodeParams: NodeParams, blockHeight: AtomicLong, client } } + private def canUseTxoSpenderIndex: Future[Boolean] = client.getIndexInfo().map(_.get("txospenderindex").exists(_.synced)) + private def checkSpent(w: WatchSpent[_ <: WatchSpentTriggered]): Future[Unit] = { // First let's see if the parent tx was published or not before checking whether it has been spent. client.getTxConfirmations(w.txId).collect { @@ -435,45 +439,60 @@ private class ZmqWatcher(nodeParams: NodeParams, blockHeight: AtomicLong, client // This is an external channels: funds are not at risk, so we don't need to scan the blockchain to find the // spending transaction, it is costly and unnecessary. We simply check whether the output has already been // spent by a confirmed transaction. - client.isTransactionOutputSpent(w.txId, w.outputIndex).collect { + client.isTransactionOutputSpent(w.outPoint).collect { case true => // The output has been spent, so we trigger the watch without including the spending transaction. context.self ! TriggerEvent(w.replyTo, w, WatchExternalChannelSpentTriggered(w.shortChannelId, None)) } case _ => // The parent tx was published, we need to make sure this particular output has not been spent. - client.isTransactionOutputSpendable(w.txId, w.outputIndex, includeMempool = true).collect { - case false => - // The output has been spent, let's find the spending tx. - // If we know some potential spending txs, we try to fetch them directly. - Future.sequence(w.hints.map(txid => client.getTransaction(txid).map(Some(_)).recover { case _ => None })) - .map(_.flatten) // filter out errors and hint transactions that can't be found - .map(hintTxs => { - hintTxs.find(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match { - case Some(spendingTx) => - log.info("{}:{} has already been spent by a tx provided in hints: txid={}", w.txId, w.outputIndex, spendingTx.txid) - context.self ! ProcessNewTransaction(spendingTx) - case None => - // The hints didn't help us, let's search for the spending transaction in the mempool. - log.info("{}:{} has already been spent, looking for the spending tx in the mempool", w.txId, w.outputIndex) - client.lookForMempoolSpendingTx(w.txId, w.outputIndex).map(Some(_)).recover { case _ => None }.map { - case Some(spendingTx) => - log.info("found tx spending {}:{} in the mempool: txid={}", w.txId, w.outputIndex, spendingTx.txid) - context.self ! ProcessNewTransaction(spendingTx) - case None => - // The spending transaction isn't in the mempool, so it must be a transaction that confirmed - // before we set the watch. We have to scan the blockchain to find it, which is expensive - // since bitcoind doesn't provide indexes for this scenario. - log.warn("{}:{} has already been spent, spending tx not in the mempool, looking in the blockchain...", w.txId, w.outputIndex) - client.lookForSpendingTx(None, w.txId, w.outputIndex, nodeParams.channelConf.maxChannelSpentRescanBlocks).map { spendingTx => - log.warn("found the spending tx of {}:{} in the blockchain: txid={}", w.txId, w.outputIndex, spendingTx.txid) + client.isTransactionOutputSpendable(w.outPoint, includeMempool = true).collect { + case false => canUseTxoSpenderIndex.foreach { + case true => + // The output has been spent, let's find the spending tx in the txospenderindex. + log.info("{} has already been spent, looking for the spending tx", w.outPoint) + client.findSpendingTx(w.outPoint) map { + case Some((spendingTx, _)) => + log.info("found tx spending {} txid={}", w.outPoint, spendingTx.txid) + context.self ! ProcessNewTransaction(spendingTx) + case None => + log.warn("could not find the spending tx of {}, funds are at risk", w.outPoint) + } recover { + case error => + log.warn("error finding the spending tx of {}, funds are at risk", w.outPoint, error) + } + case false => + // txospenderindex is not available, scan the mempool and the blockchain + // If we know some potential spending txs, we try to fetch them directly. + Future.sequence(w.hints.map(txid => client.getTransaction(txid).map(Some(_)).recover { case _ => None })) + .map(_.flatten) // filter out errors and hint transactions that can't be found + .map(hintTxs => { + hintTxs.find(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match { + case Some(spendingTx) => + log.info("{}:{} has already been spent by a tx provided in hints: txid={}", w.txId, w.outputIndex, spendingTx.txid) + context.self ! ProcessNewTransaction(spendingTx) + case None => + // The hints didn't help us, let's search for the spending transaction in the mempool. + log.info("{}:{} has already been spent, looking for the spending tx in the mempool", w.txId, w.outputIndex) + client.lookForMempoolSpendingTx(w.outPoint).map(Some(_)).recover { case _ => None }.map { + case Some(spendingTx) => + log.info("found tx spending {}:{} in the mempool: txid={}", w.txId, w.outputIndex, spendingTx.txid) context.self ! ProcessNewTransaction(spendingTx) - }.recover { - case _ => log.warn("could not find the spending tx of {}:{} in the blockchain, funds are at risk", w.txId, w.outputIndex) - } - } - } - }) + case None => + // The spending transaction isn't in the mempool, so it must be a transaction that confirmed + // before we set the watch. We have to scan the blockchain to find it, which is expensive + // since bitcoind doesn't provide indexes for this scenario. + log.warn("{}:{} has already been spent, spending tx not in the mempool, looking in the blockchain...", w.txId, w.outputIndex) + client.lookForSpendingTx(None, w.outPoint, nodeParams.channelConf.maxChannelSpentRescanBlocks).map { spendingTx => + log.warn("found the spending tx of {}:{} in the blockchain: txid={}", w.txId, w.outputIndex, spendingTx.txid) + context.self ! ProcessNewTransaction(spendingTx) + }.recover { + case _ => log.warn("could not find the spending tx of {}:{} in the blockchain, funds are at risk", w.txId, w.outputIndex) + } + } + } + }) + } } } } diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala index 0a93000e88..6babf92f19 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala @@ -116,17 +116,17 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool * (not in the blockchain nor in the mempool) but could reappear later and be spendable at that point. If you want to * ensure that an output is not spendable anymore, you should use [[isTransactionOutputSpent]]. */ - def isTransactionOutputSpendable(txid: TxId, outputIndex: Int, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] = + def isTransactionOutputSpendable(outPoint: OutPoint, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] = for { - json <- rpcClient.invoke("gettxout", txid, outputIndex, includeMempool) + json <- rpcClient.invoke("gettxout", outPoint.txid, outPoint.index, includeMempool) } yield json != JNull /** * Return true if this output has already been spent by a confirmed transaction. * Note that a reorg may invalidate the result of this function and make a spent output spendable again. */ - def isTransactionOutputSpent(txid: TxId, outputIndex: Int)(implicit ec: ExecutionContext): Future[Boolean] = { - getTxConfirmations(txid).flatMap { + def isTransactionOutputSpent(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Boolean] = { + getTxConfirmations(outPoint.txid).flatMap { case Some(confirmations) if confirmations > 0 => // There is an important limitation when using isTransactionOutputSpendable: if it returns false, it can mean a // few different things: @@ -137,7 +137,7 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool // The only way to make sure that our output has been spent is to verify that it is coming from a confirmed // transaction and that it has been spent by another confirmed transaction. We want to ignore the mempool to // only consider spending transactions that have been confirmed. - isTransactionOutputSpendable(txid, outputIndex, includeMempool = false).map(r => !r) + isTransactionOutputSpendable(outPoint, includeMempool = false).map(r => !r) case _ => // If the output itself isn't in the blockchain, it cannot be spent by a confirmed transaction. Future.successful(false) @@ -166,34 +166,50 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool // themselves been double-spent, we will never be able to consider our transaction double-spent. With the // information we have, these unknown inputs could eventually reappear and the transaction could be broadcast // again. - Future.sequence(tx.txIn.map(txIn => isTransactionOutputSpent(txIn.outPoint.txid, txIn.outPoint.index.toInt))).map(_.exists(_ == true)) + Future.sequence(tx.txIn.map(txIn => isTransactionOutputSpent(txIn.outPoint))).map(_.exists(_ == true)) } } yield doubleSpent /** Search for mempool transaction spending a given output. */ - def lookForMempoolSpendingTx(txid: TxId, outputIndex: Int)(implicit ec: ExecutionContext): Future[Transaction] = { - rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(txid, outputIndex))).collect { + def lookForMempoolSpendingTx(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Transaction] = { + val options = JObject(List("return_spending_tx" -> JBool(true), "mempool_only" -> JBool(true))) + rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(outPoint.txid, outPoint.index)), options).collect { case JArray(results) => results.flatMap(result => (result \ "spendingtxid").extractOpt[String].map(TxId.fromValidHex)) }.flatMap { spendingTxIds => spendingTxIds.headOption match { case Some(spendingTxId) => getTransaction(spendingTxId) - case None => Future.failed(new RuntimeException(s"mempool doesn't contain any transaction spending $txid:$outputIndex")) + case None => Future.failed(new RuntimeException(s"mempool doesn't contain any transaction spending $outPoint")) } } } + /** + * Find the transaction spending a given output. Requires `txospenderindex` on the bitcoin code node we're connecting to. + * + * @param outPoint transaction output + * @return the transaction that spent this output along with the id of the block it was published in if any, or None if no spending transaction was found + */ + def findSpendingTx(outPoint: OutPoint)(implicit ec: ExecutionContext): Future[Option[(Transaction, Option[BlockId])]] = { + val options = JObject(List("return_spending_tx" -> JBool(true))) + rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(outPoint.txid, outPoint.index)), options).collect { + case JArray(results) => results.flatMap(result => { + val tx_opt = (result \ "spendingtx").extractOpt[String].map(Transaction.read) + tx_opt.map(tx => tx -> (result \ "blockhash").extractOpt[String].map(s => BlockId(ByteVector32.fromValidHex(s)))) + }).headOption + } + } + /** * Iterate over blocks to find the transaction that has spent a given output. * It isn't useful to look at the whole blockchain history: if the transaction was confirmed long ago, an attacker * will have already claimed all possible outputs and there's nothing we can do about it. * * @param blockHash_opt hash of a block *after* the output has been spent. If not provided, we will use the blockchain tip. - * @param txid id of the transaction output that has been spent. - * @param outputIndex index of the transaction output that has been spent. + * @param outPoint transaction output that has been spent. * @param limit maximum number of previous blocks to scan. * @return the transaction spending the given output. */ - def lookForSpendingTx(blockHash_opt: Option[BlockHash], txid: TxId, outputIndex: Int, limit: Int)(implicit ec: ExecutionContext): Future[Transaction] = { + def lookForSpendingTx(blockHash_opt: Option[BlockHash], outPoint: OutPoint, limit: Int)(implicit ec: ExecutionContext): Future[Transaction] = { for { blockId <- blockHash_opt match { case Some(blockHash) => Future.successful(BlockId(blockHash)) @@ -201,10 +217,10 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool case None => rpcClient.invoke("getbestblockhash").collect { case JString(blockId) => BlockId(ByteVector32.fromValidHex(blockId)) } } block <- getBlock(blockId) - res <- block.tx.asScala.find(tx => tx.txIn.asScala.exists(i => i.outPoint.txid == KotlinUtils.scala2kmp(txid) && i.outPoint.index == outputIndex)) match { + res <- block.tx.asScala.find(tx => tx.txIn.asScala.exists(i => i.outPoint == KotlinUtils.scala2kmp(outPoint))) match { case Some(tx) => Future.successful(KotlinUtils.kmp2scala(tx)) - case None if limit > 0 => lookForSpendingTx(Some(KotlinUtils.kmp2scala(block.header.hashPreviousBlock)), txid, outputIndex, limit - 1) - case None => Future.failed(new RuntimeException(s"couldn't find tx spending $txid:$outputIndex in the blockchain")) + case None if limit > 0 => lookForSpendingTx(Some(KotlinUtils.kmp2scala(block.header.hashPreviousBlock)), outPoint, limit - 1) + case None => Future.failed(new RuntimeException(s"couldn't find tx spending $outPoint in the blockchain")) } } yield res } @@ -755,12 +771,13 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool TxId.fromValidHex(txs(txIndex).extract[String]) }.getOrElse(TxId(ByteVector32.Zeroes))) tx <- getRawTransaction(txid) - unspent <- isTransactionOutputSpendable(txid, outputIndex, includeMempool = true) + outPoint = OutPoint(txid, outputIndex) + unspent <- isTransactionOutputSpendable(outPoint, includeMempool = true) fundingTxStatus <- if (unspent) { Future.successful(UtxoStatus.Unspent) } else { // if this returns true, it means that the spending tx is *not* in the blockchain - isTransactionOutputSpendable(txid, outputIndex, includeMempool = false).map(res => UtxoStatus.Spent(spendingTxConfirmed = !res)) + isTransactionOutputSpendable(outPoint, includeMempool = false).map(res => UtxoStatus.Spent(spendingTxConfirmed = !res)) } } yield ValidateResult(c, Right((Transaction.read(tx), fundingTxStatus))) } recover { @@ -794,6 +811,15 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val lockUtxos: Bool }) } + //------------------------- MISC -------------------------// + + /** + * + * @return information about enabled bitcoin core indexes, in a map where the key is the index name + */ + def getIndexInfo()(implicit ec: ExecutionContext): Future[Map[String, IndexInfo]] = rpcClient.invoke("getindexinfo").collect { + case JObject(results) => results.map { case (name, o) => name -> BitcoinCoreClient.IndexInfo((o \ "synced").extract[Boolean], (o \ "best_block_height").extract[Int]) }.toMap + } } object BitcoinCoreClient { @@ -871,4 +897,11 @@ object BitcoinCoreClient { // @formatter:on } + /** + * Information about a bitcoin core inedx + * + * @param synced true if the index is synced + * @param bestBlockHeight height of the last indexed block + */ + case class IndexInfo(synced: Boolean, bestBlockHeight: Int) } \ No newline at end of file diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/channel/fund/InteractiveTxBuilder.scala b/eclair-core/src/main/scala/fr/acinq/eclair/channel/fund/InteractiveTxBuilder.scala index b054ecc980..a82ee08d6b 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/channel/fund/InteractiveTxBuilder.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/channel/fund/InteractiveTxBuilder.scala @@ -758,7 +758,7 @@ private class InteractiveTxBuilder(replyTo: ActorRef[InteractiveTxBuilder.Respon // unconfirmed inputs, because if they are valid but not in our mempool we would incorrectly consider // them unspendable (unknown). We want to reject unspendable inputs to immediately fail the funding // attempt, instead of waiting to detect the double-spend later. - wallet.isTransactionOutputSpendable(outpoint.txid, outpoint.index.toInt, includeMempool = true) + wallet.isTransactionOutputSpendable(outpoint, includeMempool = true) case _ => Future.successful(false) } case Success(false) => Future.successful(false) diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/MempoolTxMonitor.scala b/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/MempoolTxMonitor.scala index df7c334063..7c5fc1a91c 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/MempoolTxMonitor.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/MempoolTxMonitor.scala @@ -200,8 +200,8 @@ private class MempoolTxMonitor(nodeParams: NodeParams, private def checkInputStatus(input: OutPoint): Unit = { val checkInputTask = for { parentConfirmations <- bitcoinClient.getTxConfirmations(input.txid) - spendableMempoolExcluded <- bitcoinClient.isTransactionOutputSpendable(input.txid, input.index.toInt, includeMempool = false) - spendableMempoolIncluded <- bitcoinClient.isTransactionOutputSpendable(input.txid, input.index.toInt, includeMempool = true) + spendableMempoolExcluded <- bitcoinClient.isTransactionOutputSpendable(input, includeMempool = false) + spendableMempoolIncluded <- bitcoinClient.isTransactionOutputSpendable(input, includeMempool = true) } yield computeInputStatus(parentConfirmations, spendableMempoolExcluded, spendableMempoolIncluded) context.pipeToSelf(checkInputTask) { case Success(status) => status diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/ReplaceableTxPrePublisher.scala b/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/ReplaceableTxPrePublisher.scala index 368cf56d0d..6a0d831044 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/ReplaceableTxPrePublisher.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/ReplaceableTxPrePublisher.scala @@ -99,7 +99,7 @@ private class ReplaceableTxPrePublisher(nodeParams: NodeParams, context.pipeToSelf(bitcoinClient.getTxConfirmations(fundingOutpoint.txid).flatMap { case Some(_) => // The funding transaction was found, let's see if we can still spend it. - bitcoinClient.isTransactionOutputSpendable(fundingOutpoint.txid, fundingOutpoint.index.toInt, includeMempool = true).flatMap { + bitcoinClient.isTransactionOutputSpendable(fundingOutpoint, includeMempool = true).flatMap { case true => // The funding output is unspent: let's publish our anchor transaction to get our local commit confirmed. Future.successful(ParentTxOk) @@ -162,7 +162,7 @@ private class ReplaceableTxPrePublisher(nodeParams: NodeParams, case Some(_) => // The funding transaction was found, let's see if we can still spend it. Note that in this case, we only look // at *confirmed* spending transactions (unlike the local commit case). - bitcoinClient.isTransactionOutputSpendable(fundingOutpoint.txid, fundingOutpoint.index.toInt, includeMempool = false).flatMap { + bitcoinClient.isTransactionOutputSpendable(fundingOutpoint, includeMempool = false).flatMap { case true => // The funding output is unspent, or spent by an *unconfirmed* transaction: let's publish our anchor // transaction, we may be able to replace our local commit with this (more interesting) remote commit. @@ -221,7 +221,7 @@ private class ReplaceableTxPrePublisher(nodeParams: NodeParams, case Some(_) => // If the HTLC output is already spent by a confirmed transaction, there is no need for RBF: either this is one // of our transactions (which thus has a high enough feerate), or it was a race with our peer and we lost. - bitcoinClient.isTransactionOutputSpent(input.txid, input.index.toInt).map { + bitcoinClient.isTransactionOutputSpent(input).map { case true => HtlcOutputAlreadySpent case false => ParentTxOk } diff --git a/eclair-core/src/test/resources/integration/bitcoin.conf b/eclair-core/src/test/resources/integration/bitcoin.conf index 370f1cbfcb..13bc1b185e 100644 --- a/eclair-core/src/test/resources/integration/bitcoin.conf +++ b/eclair-core/src/test/resources/integration/bitcoin.conf @@ -4,6 +4,7 @@ server=1 rpcuser=foo rpcpassword=bar txindex=1 +txospenderindex=1 zmqpubhashblock=tcp://127.0.0.1:28334 zmqpubrawtx=tcp://127.0.0.1:28335 rpcworkqueue=64 diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/DummyOnChainWallet.scala b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/DummyOnChainWallet.scala index 7561ea0152..32cae8c807 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/DummyOnChainWallet.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/DummyOnChainWallet.scala @@ -77,7 +77,7 @@ class DummyOnChainWallet extends OnChainWallet with OnChainAddressCache { override def getTxConfirmations(txid: TxId)(implicit ec: ExecutionContext): Future[Option[Int]] = Future.failed(new RuntimeException("transaction not found")) - override def isTransactionOutputSpendable(txid: TxId, outputIndex: Int, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] = Future.successful(true) + override def isTransactionOutputSpendable(outPoint: OutPoint, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] = Future.successful(true) override def rollback(tx: Transaction)(implicit ec: ExecutionContext): Future[Boolean] = { rolledback = rolledback + tx @@ -218,7 +218,7 @@ class SingleKeyOnChainWallet extends OnChainWallet with OnChainAddressCache { override def getTxConfirmations(txid: TxId)(implicit ec: ExecutionContext): Future[Option[Int]] = Future.successful(None) - override def isTransactionOutputSpendable(txid: TxId, outputIndex: Int, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] = Future.successful(true) + override def isTransactionOutputSpendable(outPoint: OutPoint, includeMempool: Boolean)(implicit ec: ExecutionContext): Future[Boolean] = Future.successful(true) override def rollback(tx: Transaction)(implicit ec: ExecutionContext): Future[Boolean] = { rolledback = rolledback :+ tx diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala index 875a614b64..75b448b37c 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala @@ -1726,19 +1726,19 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A bitcoinClient.getTxConfirmations(tx1.txid).pipeTo(sender.ref) sender.expectMsg(Some(0)) // If we omit the mempool, tx1's input is still considered unspent. - bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt, includeMempool = false).pipeTo(sender.ref) + bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint, includeMempool = false).pipeTo(sender.ref) sender.expectMsg(true) // If we include the mempool, we see that tx1's input is now spent. - bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt, includeMempool = true).pipeTo(sender.ref) + bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint, includeMempool = true).pipeTo(sender.ref) sender.expectMsg(false) // If we omit the mempool, tx1's output is not considered spendable because we can't even find that output. - bitcoinClient.isTransactionOutputSpendable(tx1.txid, 0, includeMempool = false).pipeTo(sender.ref) + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = false).pipeTo(sender.ref) sender.expectMsg(false) // If we include the mempool, we see that tx1 produces an output that is still unspent. - bitcoinClient.isTransactionOutputSpendable(tx1.txid, 0, includeMempool = true).pipeTo(sender.ref) + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = true).pipeTo(sender.ref) sender.expectMsg(true) // We're able to find the spending transaction in the mempool. - bitcoinClient.lookForMempoolSpendingTx(tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt).pipeTo(sender.ref) + bitcoinClient.lookForMempoolSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref) sender.expectMsg(tx1) // Let's confirm our transaction. @@ -1748,20 +1748,85 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A assert(blockHeight1 == blockHeight + 1) bitcoinClient.getTxConfirmations(tx1.txid).pipeTo(sender.ref) sender.expectMsg(Some(1)) - bitcoinClient.isTransactionOutputSpendable(tx1.txid, 0, includeMempool = false).pipeTo(sender.ref) + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = false).pipeTo(sender.ref) sender.expectMsg(true) - bitcoinClient.isTransactionOutputSpendable(tx1.txid, 0, includeMempool = true).pipeTo(sender.ref) + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = true).pipeTo(sender.ref) sender.expectMsg(true) generateBlocks(10) - bitcoinClient.lookForMempoolSpendingTx(tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt).pipeTo(sender.ref) + bitcoinClient.lookForMempoolSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref) sender.expectMsgType[Failure] - bitcoinClient.lookForSpendingTx(None, tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt, limit = 5).pipeTo(sender.ref) + bitcoinClient.lookForSpendingTx(None, tx1.txIn.head.outPoint, limit = 5).pipeTo(sender.ref) sender.expectMsgType[Failure] - bitcoinClient.lookForSpendingTx(None, tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt, limit = 15).pipeTo(sender.ref) + bitcoinClient.lookForSpendingTx(None, tx1.txIn.head.outPoint, limit = 15).pipeTo(sender.ref) sender.expectMsg(tx1) } + test("get index information") { + val sender = TestProbe() + val bitcoinClient = makeBitcoinCoreClient() + bitcoinClient.getIndexInfo().pipeTo(sender.ref) + val indexInfos = sender.expectMsgType[Map[String, BitcoinCoreClient.IndexInfo]] + assert(indexInfos("txindex").synced) + assert(indexInfos("txospenderindex").synced) + } + + test("find spending transaction of a given output using txospenderindex") { + val sender = TestProbe() + val bitcoinClient = makeBitcoinCoreClient() + + bitcoinClient.getBlockHeight().pipeTo(sender.ref) + val blockHeight = sender.expectMsgType[BlockHeight] + + bitcoinClient.getIndexInfo().pipeTo(sender.ref) + val indexInfos = sender.expectMsgType[Map[String, BitcoinCoreClient.IndexInfo]] + assert(indexInfos("txindex").synced) + + val address = getNewAddress(sender) + val tx1 = sendToAddress(address, 5 btc) + + // Transaction is still in the mempool at that point + bitcoinClient.getTxConfirmations(tx1.txid).pipeTo(sender.ref) + sender.expectMsg(Some(0)) + // If we omit the mempool, tx1's input is still considered unspent. + bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint, includeMempool = false).pipeTo(sender.ref) + sender.expectMsg(true) + // If we include the mempool, we see that tx1's input is now spent. + bitcoinClient.isTransactionOutputSpendable(tx1.txIn.head.outPoint, includeMempool = true).pipeTo(sender.ref) + sender.expectMsg(false) + // If we omit the mempool, tx1's output is not considered spendable because we can't even find that output. + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = false).pipeTo(sender.ref) + sender.expectMsg(false) + // If we include the mempool, we see that tx1 produces an output that is still unspent. + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = true).pipeTo(sender.ref) + sender.expectMsg(true) + // We're able to find the spending transaction in the mempool. + bitcoinClient.findSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref) + sender.expectMsg(Some(tx1, None)) + + // Let's confirm our transaction. + generateBlocks(1) + bitcoinClient.getBlockHeight().pipeTo(sender.ref) + val blockHeight1 = sender.expectMsgType[BlockHeight] + assert(blockHeight1 == blockHeight + 1) + bitcoinClient.getBlockId(blockHeight1.toInt).pipeTo(sender.ref) + val tip = sender.expectMsgType[BlockId] + bitcoinClient.findSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref) + sender.expectMsg(Some(tx1, Some(tip))) + + bitcoinClient.getTxConfirmations(tx1.txid).pipeTo(sender.ref) + sender.expectMsg(Some(1)) + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = false).pipeTo(sender.ref) + sender.expectMsg(true) + bitcoinClient.isTransactionOutputSpendable(OutPoint(tx1.txid, 0), includeMempool = true).pipeTo(sender.ref) + sender.expectMsg(true) + + // we still fid the spending tx evn if it has been confirmed many times + generateBlocks(10) + bitcoinClient.findSpendingTx(tx1.txIn.head.outPoint).pipeTo(sender.ref) + sender.expectMsg(Some(tx1, Some(tip))) + } + test("get pubkey for p2wpkh receive address") { val sender = TestProbe() val bitcoinClient = makeBitcoinCoreClient() @@ -1978,7 +2043,7 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A // Both funding outputs have been spent by transactions that are external to our wallet. fundingUtxos.foreach { utxo => - wallet.isTransactionOutputSpendable(utxo.txid, utxo.index.toInt, includeMempool = false).pipeTo(sender.ref) + wallet.isTransactionOutputSpendable(utxo, includeMempool = false).pipeTo(sender.ref) sender.expectMsg(false) } Seq(remoteCommitTx1, remoteCommitTx2).foreach { tx => diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala index 6ca2a7a030..6dbf8680a5 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala @@ -62,7 +62,7 @@ trait BitcoindService extends Logging { val PATH_BITCOIND = sys.env.get("BITCOIND_DIR") match { case Some(customBitcoinDir) => new File(customBitcoinDir, "bitcoind") - case None => new File(TestUtils.BUILD_DIRECTORY, "bitcoin-30.2/bin/bitcoind") + case None => new File(TestUtils.BUILD_DIRECTORY, "bitcoin-31.0/bin/bitcoind") } logger.info(s"using bitcoind: $PATH_BITCOIND") val PATH_BITCOIND_DATADIR = new File(INTEGRATION_TMP_DIR, "datadir-bitcoin") diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcherSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcherSpec.scala index f55b3cbbd7..72ba26b346 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcherSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcherSpec.scala @@ -72,7 +72,7 @@ class ZmqWatcherSpec extends TestKitBaseClass with AnyFunSuiteLike with Bitcoind case class Fixture(blockHeight: AtomicLong, bitcoinClient: BitcoinCoreClient, watcher: typed.ActorRef[ZmqWatcher.Command], probe: TestProbe, listener: TestProbe) // NB: we can't use ScalaTest's fixtures, they would see uninitialized bitcoind fields because they sandbox each test. - private def withWatcher(testFun: Fixture => Any, scanPastBlock: Boolean = false): Unit = { + private def withWatcher(testFun: Fixture => Any, scanPastBlock: Boolean = false, useTxoSpenderIndex: Boolean = false): Unit = { val blockCount = new AtomicLong() val probe = TestProbe() val listener = TestProbe() @@ -85,6 +85,7 @@ class ZmqWatcherSpec extends TestKitBaseClass with AnyFunSuiteLike with Bitcoind // We enable it and use a faster (randomized) delay when requested. .modify(_.channelConf.scanPreviousBlocksDepth).setToIf(scanPastBlock)(6) .modify(_.channelConf.maxBlockProcessingDelay).setToIf(scanPastBlock)(10 millis) + //.modify(_.channelConf.useTxoSpenderIndex).setTo(useTxoSpenderIndex) val watcher = system.spawn(ZmqWatcher(nodeParams, blockCount, bitcoinClient), UUID.randomUUID().toString) try { testFun(Fixture(blockCount, bitcoinClient, watcher, probe, listener)) @@ -245,80 +246,86 @@ class ZmqWatcherSpec extends TestKitBaseClass with AnyFunSuiteLike with Bitcoind }) } - test("watch for spent transactions") { - withWatcher(f => { - import f._ - - val (priv, address) = createExternalAddress() - val tx = sendToAddress(address, Btc(1), probe) - val outputIndex = tx.txOut.indexWhere(_.publicKeyScript == Script.write(Script.pay2wpkh(priv.publicKey))) - val (tx1, tx2) = createUnspentTxChain(tx, priv) - - watcher ! WatchExternalChannelSpent(probe.ref, tx.txid, outputIndex, RealShortChannelId(5)) - watcher ! WatchFundingSpent(probe.ref, tx.txid, outputIndex, Set.empty) - probe.expectNoMessage(100 millis) - - watcher ! ListWatches(probe.ref) - assert(probe.expectMsgType[Set[Watch[_]]].size == 2) - - bitcoinClient.publishTransaction(tx1) - // tx and tx1 aren't confirmed yet, but we trigger the WatchSpentTriggered event when we see tx1 in the mempool. - probe.expectMsgAllOf( - WatchExternalChannelSpentTriggered(RealShortChannelId(5), Some(tx1)), - WatchFundingSpentTriggered(tx1) - ) - // Let's confirm tx and tx1: seeing tx1 in a block should trigger both WatchSpentTriggered events again. - bitcoinClient.getBlockHeight().pipeTo(probe.ref) - val initialBlockHeight = probe.expectMsgType[BlockHeight] - generateBlocks(1) - probe.expectMsgAllOf( - WatchExternalChannelSpentTriggered(RealShortChannelId(5), Some(tx1)), - WatchFundingSpentTriggered(tx1) - ) - probe.expectNoMessage(100 millis) - - watcher ! ListWatches(probe.ref) - val watches1 = probe.expectMsgType[Set[Watch[_]]] - assert(watches1.size == 2) - assert(watches1.forall(_.isInstanceOf[WatchSpent[_]])) - - // Let's submit tx2, and set a watch after it has been confirmed this time. - bitcoinClient.publishTransaction(tx2) - probe.expectNoMessage(100 millis) + def watchForSpentTransactions(f: Fixture): Unit = { + import f._ + + val (priv, address) = createExternalAddress() + val tx = sendToAddress(address, Btc(1), probe) + val outputIndex = tx.txOut.indexWhere(_.publicKeyScript == Script.write(Script.pay2wpkh(priv.publicKey))) + val (tx1, tx2) = createUnspentTxChain(tx, priv) + + watcher ! WatchExternalChannelSpent(probe.ref, tx.txid, outputIndex, RealShortChannelId(5)) + watcher ! WatchFundingSpent(probe.ref, tx.txid, outputIndex, Set.empty) + probe.expectNoMessage(100 millis) + + watcher ! ListWatches(probe.ref) + assert(probe.expectMsgType[Set[Watch[_]]].size == 2) + + bitcoinClient.publishTransaction(tx1) + // tx and tx1 aren't confirmed yet, but we trigger the WatchSpentTriggered event when we see tx1 in the mempool. + probe.expectMsgAllOf( + WatchExternalChannelSpentTriggered(RealShortChannelId(5), Some(tx1)), + WatchFundingSpentTriggered(tx1) + ) + // Let's confirm tx and tx1: seeing tx1 in a block should trigger both WatchSpentTriggered events again. + bitcoinClient.getBlockHeight().pipeTo(probe.ref) + val initialBlockHeight = probe.expectMsgType[BlockHeight] + generateBlocks(1) + probe.expectMsgAllOf( + WatchExternalChannelSpentTriggered(RealShortChannelId(5), Some(tx1)), + WatchFundingSpentTriggered(tx1) + ) + probe.expectNoMessage(100 millis) + + watcher ! ListWatches(probe.ref) + val watches1 = probe.expectMsgType[Set[Watch[_]]] + assert(watches1.size == 2) + assert(watches1.forall(_.isInstanceOf[WatchSpent[_]])) + + // Let's submit tx2, and set a watch after it has been confirmed this time. + bitcoinClient.publishTransaction(tx2) + probe.expectNoMessage(100 millis) + + system.eventStream.subscribe(probe.ref, classOf[CurrentBlockHeight]) + generateBlocks(1) + awaitCond(probe.expectMsgType[CurrentBlockHeight].blockHeight >= initialBlockHeight + 2) + + watcher ! ListWatches(probe.ref) + val watches2 = probe.expectMsgType[Set[Watch[_]]] + assert(watches2.size == 2) + assert(watches2.forall(_.isInstanceOf[WatchSpent[_]])) + watcher ! StopWatching(probe.ref) + + // We use hints and see if we can find tx2 + watcher ! WatchFundingSpent(probe.ref, tx1.txid, 0, Set(tx2.txid)) + probe.expectMsg(WatchFundingSpentTriggered(tx2)) + watcher ! StopWatching(probe.ref) + + // We should still find tx2 if the provided hint is wrong + watcher ! WatchOutputSpent(probe.ref, tx1.txid, 0, tx1.txOut(0).amount, Set(randomTxId())) + probe.fishForMessage() { case m: WatchOutputSpentTriggered => m.spendingTx.txid == tx2.txid } + watcher ! StopWatching(probe.ref) + + // We should find txs that have already been confirmed + watcher ! WatchOutputSpent(probe.ref, tx.txid, outputIndex, tx.txOut(outputIndex).amount, Set.empty) + probe.fishForMessage() { case m: WatchOutputSpentTriggered => m.spendingTx.txid == tx1.txid } + watcher ! StopWatching(probe.ref) + + // If we watch after being spent by a confirmed transaction, we immediately trigger the watch without fetching + // the spending transaction. + watcher ! WatchExternalChannelSpent(probe.ref, tx1.txid, 0, RealShortChannelId(1)) + probe.expectMsg(WatchExternalChannelSpentTriggered(RealShortChannelId(1), None)) + watcher ! StopWatching(probe.ref) + watcher ! WatchFundingSpent(probe.ref, tx1.txid, 0, Set.empty) + probe.expectMsg(WatchFundingSpentTriggered(tx2)) + } - system.eventStream.subscribe(probe.ref, classOf[CurrentBlockHeight]) - generateBlocks(1) - awaitCond(probe.expectMsgType[CurrentBlockHeight].blockHeight >= initialBlockHeight + 2) + test("watch for spent transactions") { + withWatcher(f => watchForSpentTransactions(f)) + } - watcher ! ListWatches(probe.ref) - val watches2 = probe.expectMsgType[Set[Watch[_]]] - assert(watches2.size == 2) - assert(watches2.forall(_.isInstanceOf[WatchSpent[_]])) - watcher ! StopWatching(probe.ref) - - // We use hints and see if we can find tx2 - watcher ! WatchFundingSpent(probe.ref, tx1.txid, 0, Set(tx2.txid)) - probe.expectMsg(WatchFundingSpentTriggered(tx2)) - watcher ! StopWatching(probe.ref) - - // We should still find tx2 if the provided hint is wrong - watcher ! WatchOutputSpent(probe.ref, tx1.txid, 0, tx1.txOut(0).amount, Set(randomTxId())) - probe.fishForMessage() { case m: WatchOutputSpentTriggered => m.spendingTx.txid == tx2.txid } - watcher ! StopWatching(probe.ref) - - // We should find txs that have already been confirmed - watcher ! WatchOutputSpent(probe.ref, tx.txid, outputIndex, tx.txOut(outputIndex).amount, Set.empty) - probe.fishForMessage() { case m: WatchOutputSpentTriggered => m.spendingTx.txid == tx1.txid } - watcher ! StopWatching(probe.ref) - - // If we watch after being spent by a confirmed transaction, we immediately trigger the watch without fetching - // the spending transaction. - watcher ! WatchExternalChannelSpent(probe.ref, tx1.txid, 0, RealShortChannelId(1)) - probe.expectMsg(WatchExternalChannelSpentTriggered(RealShortChannelId(1), None)) - watcher ! StopWatching(probe.ref) - watcher ! WatchFundingSpent(probe.ref, tx1.txid, 0, Set.empty) - probe.expectMsg(WatchFundingSpentTriggered(tx2)) - }) + test("watch for spent transactions with txospenderindex") { + withWatcher(f => watchForSpentTransactions(f), useTxoSpenderIndex = true) } test("unwatch external channel") { diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/integration/ChannelIntegrationSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/integration/ChannelIntegrationSpec.scala index 761da2b6d9..d9ca6a3cf6 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/integration/ChannelIntegrationSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/integration/ChannelIntegrationSpec.scala @@ -84,7 +84,7 @@ abstract class ChannelIntegrationSpec extends IntegrationSpec { /** Wait for the given outpoint to be spent (either by a mempool or confirmed transaction). */ def waitForOutputSpent(outpoint: OutPoint, bitcoinClient: BitcoinCoreClient, sender: TestProbe): Unit = { awaitCond({ - bitcoinClient.isTransactionOutputSpendable(outpoint.txid, outpoint.index.toInt, includeMempool = true).pipeTo(sender.ref) + bitcoinClient.isTransactionOutputSpendable(outpoint, includeMempool = true).pipeTo(sender.ref) val isSpendable = sender.expectMsgType[Boolean] !isSpendable }, max = 30 seconds, interval = 1 second)