From 850ee975f82b62b538ae12c248f77cd3e24b2ec3 Mon Sep 17 00:00:00 2001 From: Pacu Date: Mon, 27 Apr 2026 12:49:47 -0300 Subject: [PATCH 1/8] Match lightwalletd compact block responses Align Zaino's CompactTxStreamer responses with lightwalletd for single-block lookups. GetBlock and GetBlockNullifiers now request all pool data, compact blocks use lightwalletd's protoVersion value, nullifier responses clear transparent inputs and outputs, and tree-state RPCs are delegated to the backing validator so they are not limited by the local cache view. --- packages/zaino-proto/src/proto/utils.rs | 2 + packages/zaino-state/src/backends/fetch.rs | 88 +++++-------------- packages/zaino-state/src/backends/state.rs | 68 +++----------- .../finalised_state/db/v1/compact_block.rs | 4 +- .../src/chain_index/types/db/legacy.rs | 2 +- 5 files changed, 41 insertions(+), 123 deletions(-) diff --git a/packages/zaino-proto/src/proto/utils.rs b/packages/zaino-proto/src/proto/utils.rs index 6c4de0bbc..c52a5d8df 100644 --- a/packages/zaino-proto/src/proto/utils.rs +++ b/packages/zaino-proto/src/proto/utils.rs @@ -360,6 +360,8 @@ pub fn compact_block_with_pool_types( pub fn compact_block_to_nullifiers(mut block: CompactBlock) -> CompactBlock { for ctransaction in &mut block.vtx { ctransaction.outputs = Vec::new(); + ctransaction.vin = Vec::new(); + ctransaction.vout = Vec::new(); for caction in &mut ctransaction.actions { *caction = CompactOrchardAction { nullifier: caction.nullifier.clone(), diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 643d5707e..27deefdd9 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -2,7 +2,7 @@ use futures::StreamExt; use hex::FromHex; -use std::{io::Cursor, str::FromStr, time}; +use std::{io::Cursor, time}; use tokio::{sync::mpsc, time::timeout}; use tonic::async_trait; use tracing::{info, instrument, warn}; @@ -550,51 +550,8 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, hash_or_height: String, ) -> Result { - let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let block_data = match hash_or_height_struct { - HashOrHeight::Hash(hash) => self - .indexer - .get_indexed_block_by_hash(&snapshot, &hash.into()) - .await - .map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or( - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )), - )?, - HashOrHeight::Height(height) => self - .indexer - .get_indexed_block_by_height(&snapshot, &height.into()) - .await - .map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or( - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )), - )?, - }; - - let (sapling, orchard) = self - .indexer - .get_treestate(block_data.hash()) + self.fetcher + .get_treestate(hash_or_height) .await .map_err(|_error| { #[allow(deprecated)] @@ -602,23 +559,16 @@ impl ZcashIndexer for FetchServiceSubscriber { zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch treestate.", )) - })?; - let time: u32 = block_data.data().time().try_into().map_err(|_error| { - #[allow(deprecated)] - FetchServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Block time is out of range for u32.", - )) - })?; - - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( - (*block_data.hash()).into(), - block_data.height().into(), - time, - sapling, - orchard, - )) + }) + .and_then(|treestate| { + treestate.try_into().map_err(|_error| { + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to parse treestate.", + )) + }) + }) } /// Returns information about a range of Sapling or Orchard subtrees. @@ -888,7 +838,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { match self .indexer - .get_compact_block(&snapshot, types::Height(height), PoolTypeFilter::default()) + .get_compact_block( + &snapshot, + types::Height(height), + PoolTypeFilter::includes_all(), + ) .await { Ok(Some(block)) => Ok(block), @@ -963,7 +917,11 @@ impl LightWalletIndexer for FetchServiceSubscriber { }; match self .indexer - .get_compact_block(&snapshot, types::Height(height), PoolTypeFilter::default()) + .get_compact_block( + &snapshot, + types::Height(height), + PoolTypeFilter::includes_all(), + ) .await { Ok(Some(block)) => Ok(compact_block_to_nullifiers(block)), diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index 0461a2f7f..a6cb3b14a 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1374,65 +1374,23 @@ impl ZcashIndexer for StateServiceSubscriber { &self, hash_or_height: String, ) -> Result { - let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; - let snapshot = self.indexer.snapshot_nonfinalized_state().await?; - - let block_data = match hash_or_height_struct { - HashOrHeight::Hash(hash) => self - .indexer - .get_indexed_block_by_hash(&snapshot, &hash.into()) - .await - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - HashOrHeight::Height(height) => self - .indexer - .get_indexed_block_by_height(&snapshot, &height.into()) - .await - .map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )) - })? - .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Failed to fetch block data.", - )))?, - }; - - let (sapling, orchard) = self - .indexer - .get_treestate(block_data.hash()) + self.rpc_client + .get_treestate(hash_or_height) .await .map_err(|_error| { StateServiceError::RpcError(RpcError::new_from_legacycode( zebra_rpc::server::error::LegacyCode::InvalidParameter, "Failed to fetch treestate.", )) - })?; - let time: u32 = block_data.data().time().try_into().map_err(|_error| { - StateServiceError::RpcError(RpcError::new_from_legacycode( - zebra_rpc::server::error::LegacyCode::InvalidParameter, - "Block time is out of range for u32.", - )) - })?; - - #[allow(deprecated)] - Ok(GetTreestateResponse::from_parts( - (*block_data.hash()).into(), - block_data.height().into(), - time, - sapling, - orchard, - )) + }) + .and_then(|treestate| { + treestate.try_into().map_err(|_error| { + StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to parse treestate.", + )) + }) + }) } async fn get_mining_info(&self) -> Result { @@ -1869,7 +1827,7 @@ impl LightWalletIndexer for StateServiceSubscriber { match self .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::default()) + .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) .await { Ok(Some(block)) => Ok(block), @@ -1934,7 +1892,7 @@ impl LightWalletIndexer for StateServiceSubscriber { match self .indexer - .get_compact_block(&snapshot, block_height, PoolTypeFilter::default()) + .get_compact_block(&snapshot, block_height, PoolTypeFilter::includes_all()) .await { Ok(Some(block)) => Ok(compact_block_to_nullifiers(block)), diff --git a/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs b/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs index 43a8581c1..da07ab5ae 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/db/v1/compact_block.rs @@ -248,7 +248,7 @@ impl DbV1 { // ----- Construct CompactBlock ----- Ok(zaino_proto::proto::compact_formats::CompactBlock { - proto_version: 4, + proto_version: 0, height: header.context.height().0 as u64, hash: header.context.hash().0.to_vec(), prev_hash: header.context.parent_hash().0.to_vec(), @@ -1107,7 +1107,7 @@ impl DbV1 { }; let compact_block = zaino_proto::proto::compact_formats::CompactBlock { - proto_version: 4, + proto_version: 0, height: header.context.height().0 as u64, hash: header.context.hash().0.to_vec(), prev_hash: header.context.parent_hash().0.to_vec(), diff --git a/packages/zaino-state/src/chain_index/types/db/legacy.rs b/packages/zaino-state/src/chain_index/types/db/legacy.rs index f0f970b0e..865e794cf 100644 --- a/packages/zaino-state/src/chain_index/types/db/legacy.rs +++ b/packages/zaino-state/src/chain_index/types/db/legacy.rs @@ -1194,7 +1194,7 @@ impl IndexedBlock { let orchard_commitment_tree_size = self.commitment_tree_data().sizes().orchard(); zaino_proto::proto::compact_formats::CompactBlock { - proto_version: 4, + proto_version: 0, height, hash, prev_hash, From e50a469d6d96f075029ef6174e86f7f8efd8adff Mon Sep 17 00:00:00 2001 From: Pacu Date: Mon, 27 Apr 2026 12:50:04 -0300 Subject: [PATCH 2/8] Fetch compact blocks from validator on cache miss Teach the chain index to reconstruct compact blocks from the backing validator when a serviceable non-finalized height is absent from the local snapshot/finalized DB. This matches lightwalletd's validator-backed behavior for recently loaded chains and prevents GetBlock/GetBlockRange from reporting database holes while Zaino's local cache is still catching up. --- packages/zaino-state/src/chain_index.rs | 182 ++++++++++++++++++++---- 1 file changed, 157 insertions(+), 25 deletions(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 221d91727..39343b7aa 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -14,11 +14,14 @@ use crate::chain_index::non_finalised_state::ChainIndexSnapshot; use crate::chain_index::source::GetTransactionLocation; use crate::chain_index::types::db::metadata::MempoolInfo; +use crate::chain_index::types::helpers::{BlockMetadata, BlockWithMetadata, TreeRootData}; use crate::chain_index::types::BlockIndex; use crate::chain_index::types::{BestChainLocation, NonBestChainLocation}; use crate::error::{ChainIndexError, ChainIndexErrorKind, FinalisedStateError}; use crate::status::Status; -use crate::{CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError}; +use crate::{ + ChainWork, CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError, +}; use crate::{IndexedBlock, TransactionHash}; use std::collections::HashSet; use std::{sync::Arc, time::Duration}; @@ -801,6 +804,78 @@ pub struct NodeBackedChainIndexSubscriber( + source: &Source, + network: ZebraNetwork, + height: types::Height, + pool_types: &PoolTypeFilter, +) -> Result, ChainIndexError> { + let Some(block) = source + .get_block(HashOrHeight::Height(zebra_chain::block::Height(height.0))) + .await + .map_err(ChainIndexError::backing_validator)? + else { + return Ok(None); + }; + + let block_height = block + .coinbase_height() + .map(|height| types::Height(height.0)) + .ok_or_else(|| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "validator returned a block without a height", + )) + })?; + if block_height != height { + return Err(ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "validator returned block at height {}, expected {}", + block_height.0, height.0 + ), + ))); + } + + let tree_roots = source + .get_commitment_tree_roots(types::BlockHash::from(block.hash())) + .await + .map_err(ChainIndexError::backing_validator)?; + let (sapling_root, sapling_size, orchard_root, orchard_size) = + TreeRootData::new(tree_roots.0, tree_roots.1).extract_with_defaults(); + + let metadata = BlockMetadata::new( + sapling_root, + sapling_size.try_into().map_err(|_| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sapling commitment tree size overflow", + )) + })?, + orchard_root, + orchard_size.try_into().map_err(|_| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "orchard commitment tree size overflow", + )) + })?, + ChainWork::from_u256(0.into()), + network, + ); + let indexed_block = + IndexedBlock::try_from(BlockWithMetadata::new(&block, metadata)).map_err(|error| { + ChainIndexError::backing_validator(std::io::Error::new( + std::io::ErrorKind::InvalidData, + error, + )) + })?; + + Ok(Some(compact_block_with_pool_types( + indexed_block.to_compact_block(), + &pool_types.to_pool_types_vector(), + ))) +} + impl NodeBackedChainIndexSubscriber { fn source(&self) -> &Source { &self.source @@ -834,6 +909,14 @@ impl NodeBackedChainIndexSubscriber { .transpose() } + async fn get_compact_block_from_node( + &self, + height: types::Height, + pool_types: &PoolTypeFilter, + ) -> Result, ChainIndexError> { + compact_block_from_source(self.source(), self.network.clone(), height, pool_types).await + } + async fn get_indexed_block_height( &self, snapshot: &NonfinalizedBlockCacheSnapshot, @@ -1230,19 +1313,19 @@ impl ChainIndex for NodeBackedChainIndexSubscriber match self - .finalized_state - .get_compact_block(height, pool_types) - .await - { - Ok(block) => block, - Err(e) => { - return Err(ChainIndexError::database_hole( - height, - Some(Box::new(e)), - )) + None => { + match self + .finalized_state + .get_compact_block(height, pool_types.clone()) + .await + { + Ok(block) => block, + Err(_) => self + .get_compact_block_from_node(height, &pool_types) + .await? + .ok_or(ChainIndexError::database_hole(height, None))?, } - }, + } })) } else { Ok(None) @@ -1357,6 +1440,9 @@ impl ChainIndex for NodeBackedChainIndexSubscriber ChainIndex for NodeBackedChainIndexSubscriber { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; + } + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } }; let compact_block = compact_block_with_pool_types( indexed_block.to_compact_block(), @@ -1413,12 +1522,35 @@ impl ChainIndex for NodeBackedChainIndexSubscriber { + if channel_sender.send(Ok(compact_block)).await.is_err() { + return; + } + continue; + } + Ok(None) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(format!( + "Internal error, missing nonfinalized block at height [{height_value}].", + )))) + .await; + return; + } + Err(error) => { + let _ = channel_sender + .send(Err(tonic::Status::internal(error.to_string()))) + .await; + return; + } + } }; let compact_block = compact_block_with_pool_types( indexed_block.to_compact_block(), From e09ff2e7b4abcac067518b82821d21c09705c109 Mon Sep 17 00:00:00 2001 From: Pacu Date: Mon, 27 Apr 2026 15:15:54 -0300 Subject: [PATCH 3/8] Prefer local treestate before validator fallback Restore local chain-index tree-state lookup as the primary path, and only fall back to the backing validator when the local data cannot serve the request. This keeps Zaino's local-data semantics while preserving the boot/cache-miss behavior needed for lightwalletd parity in the gRPC comparison test. --- packages/zaino-state/src/backends/fetch.rs | 58 +++++++++++++++++++++- packages/zaino-state/src/backends/state.rs | 49 +++++++++++++++++- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/packages/zaino-state/src/backends/fetch.rs b/packages/zaino-state/src/backends/fetch.rs index 27deefdd9..11ff9aca5 100644 --- a/packages/zaino-state/src/backends/fetch.rs +++ b/packages/zaino-state/src/backends/fetch.rs @@ -2,7 +2,7 @@ use futures::StreamExt; use hex::FromHex; -use std::{io::Cursor, time}; +use std::{io::Cursor, str::FromStr, time}; use tokio::{sync::mpsc, time::timeout}; use tonic::async_trait; use tracing::{info, instrument, warn}; @@ -550,8 +550,62 @@ impl ZcashIndexer for FetchServiceSubscriber { &self, hash_or_height: String, ) -> Result { + let fallback_hash_or_height = hash_or_height.clone(); + let local_result: Result = async { + let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + + let block_data = match hash_or_height_struct { + HashOrHeight::Hash(hash) => self + .indexer + .get_indexed_block_by_hash(&snapshot, &hash.into()) + .await? + .ok_or( + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )), + )?, + HashOrHeight::Height(height) => self + .indexer + .get_indexed_block_by_height(&snapshot, &height.into()) + .await? + .ok_or( + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )), + )?, + }; + + let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; + let time: u32 = block_data.data().time().try_into().map_err(|_error| { + #[allow(deprecated)] + FetchServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Block time is out of range for u32.", + )) + })?; + + #[allow(deprecated)] + Ok(GetTreestateResponse::from_parts( + (*block_data.hash()).into(), + block_data.height().into(), + time, + sapling, + orchard, + )) + } + .await; + + if let Ok(response) = local_result { + return Ok(response); + } + self.fetcher - .get_treestate(hash_or_height) + .get_treestate(fallback_hash_or_height) .await .map_err(|_error| { #[allow(deprecated)] diff --git a/packages/zaino-state/src/backends/state.rs b/packages/zaino-state/src/backends/state.rs index a6cb3b14a..6e56ad712 100644 --- a/packages/zaino-state/src/backends/state.rs +++ b/packages/zaino-state/src/backends/state.rs @@ -1374,8 +1374,55 @@ impl ZcashIndexer for StateServiceSubscriber { &self, hash_or_height: String, ) -> Result { + let fallback_hash_or_height = hash_or_height.clone(); + let local_result: Result = async { + let hash_or_height_struct: HashOrHeight = HashOrHeight::from_str(&hash_or_height)?; + let snapshot = self.indexer.snapshot_nonfinalized_state().await?; + + let block_data = match hash_or_height_struct { + HashOrHeight::Hash(hash) => self + .indexer + .get_indexed_block_by_hash(&snapshot, &hash.into()) + .await? + .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )))?, + HashOrHeight::Height(height) => self + .indexer + .get_indexed_block_by_height(&snapshot, &height.into()) + .await? + .ok_or(StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Failed to fetch block data.", + )))?, + }; + + let (sapling, orchard) = self.indexer.get_treestate(block_data.hash()).await?; + let time: u32 = block_data.data().time().try_into().map_err(|_error| { + StateServiceError::RpcError(RpcError::new_from_legacycode( + zebra_rpc::server::error::LegacyCode::InvalidParameter, + "Block time is out of range for u32.", + )) + })?; + + #[allow(deprecated)] + Ok(GetTreestateResponse::from_parts( + (*block_data.hash()).into(), + block_data.height().into(), + time, + sapling, + orchard, + )) + } + .await; + + if let Ok(response) = local_result { + return Ok(response); + } + self.rpc_client - .get_treestate(hash_or_height) + .get_treestate(fallback_hash_or_height) .await .map_err(|_error| { StateServiceError::RpcError(RpcError::new_from_legacycode( From 4d3b2bb37e61119e418d54c74db201efd788ccef Mon Sep 17 00:00:00 2001 From: zancas Date: Mon, 25 May 2026 16:42:37 -0700 Subject: [PATCH 4/8] add README for zainod --- packages/zainod/README.md | 120 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 packages/zainod/README.md diff --git a/packages/zainod/README.md b/packages/zainod/README.md new file mode 100644 index 000000000..262d8db67 --- /dev/null +++ b/packages/zainod/README.md @@ -0,0 +1,120 @@ +# zainod + +`zainod` is the Zaino indexer daemon — an indexer for the Zcash blockchain, +written in Rust. + +It sits between a Zcash full validator (Zebra or Zcashd) and client +applications, serving: + +- the **light-wallet gRPC API** (`CompactTxStreamer`), the interface today + served by [lightwalletd](https://github.com/zcash/lightwalletd), and +- a **JSON-RPC API** covering the subset of Zcash RPCs needed by wallets and + block explorers. + +This crate ships the `zainod` binary. The library half of the crate, +`zainodlib`, exposes the `run` entrypoint and configuration types for embedding +the daemon in other Rust programs. + +For project background and architecture, see the +[Zaino repository](https://github.com/zingolabs/zaino). + +## CLI + +```text +zainod generate-config [--output FILE] # write a default config file +zainod start [--config FILE] # start the indexer +``` + +When `--config`/`--output` is omitted, the path defaults to +`$XDG_CONFIG_HOME/zaino/zainod.toml` (falling back to +`$HOME/.config/zaino/zainod.toml`). + +Configuration is layered, highest priority first: + +1. environment variables (prefix `ZAINO_`), +2. the TOML config file, +3. built-in defaults. + +Sensitive fields (passwords, secrets, tokens, cookies, private keys) cannot be +set via environment variables and must come from the config file. + +## Launching + +`zainod` needs a running validator to connect to. The examples below assume one +is reachable at the address in your config. + +### From crates.io + +```sh +cargo install zainod +zainod generate-config # writes the default config, then edit it +zainod start # uses the default config path +# or point at an explicit file: +zainod start --config ./zainod.toml +``` + +### From source + +```sh +git clone https://github.com/zingolabs/zaino.git +cd zaino +cargo run --release -p zainod -- start --config ./zainod.toml +``` + +### With Podman (rootless) + +The daemon is published as a container image with `zainod start` as the default +command. It runs as a non-root user (UID 1000) and refuses to start as root, +which makes it a natural fit for rootless Podman. + +Run it directly, mounting a config file and a data volume: + +```sh +podman run --rm \ + -p 8137:8137 \ + -p 8237:8237 \ + -v ./zainod.toml:/app/config/zainod.toml:ro,Z \ + -v zaino-data:/app/data \ + zainod:latest +``` + +`--userns=keep-id` maps the container's UID 1000 to your host user, so files in +the mounted data volume stay owned by you: + +```sh +podman run --rm --userns=keep-id \ + -p 8137:8137 \ + -v ./zainod.toml:/app/config/zainod.toml:ro,Z \ + -v zaino-data:/app/data \ + zainod:latest +``` + +A typical deployment runs `zainod` alongside Zebra with `podman compose`: + +```yaml +services: + zaino: + image: zainod:latest + ports: + - "8137:8137" # gRPC + - "8237:8237" # JSON-RPC (if enabled) + volumes: + - ./config:/app/config:ro,Z + - zaino-data:/app/data + depends_on: + - zebra + +volumes: + zaino-data: +``` + +```sh +podman compose up +``` + +See [`docs/docker.md`](https://github.com/zingolabs/zaino/blob/dev/docs/docker.md) +for the full container guide. + +## License + +Apache-2.0. From 28da235799b4e6ce4b3551e1d6db5f6b24003f4c Mon Sep 17 00:00:00 2001 From: Za Wil Date: Tue, 26 May 2026 14:50:08 -0700 Subject: [PATCH 5/8] Update packages/zainod/README.md Co-authored-by: Pacu --- packages/zainod/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zainod/README.md b/packages/zainod/README.md index 262d8db67..100060bd0 100644 --- a/packages/zainod/README.md +++ b/packages/zainod/README.md @@ -6,7 +6,7 @@ written in Rust. It sits between a Zcash full validator (Zebra or Zcashd) and client applications, serving: -- the **light-wallet gRPC API** (`CompactTxStreamer`), the interface today +- the [lightclient protocol API](https://github.com/zcash/lightwallet-protocol), the interface today served by [lightwalletd](https://github.com/zcash/lightwalletd), and - a **JSON-RPC API** covering the subset of Zcash RPCs needed by wallets and block explorers. From 63af4f408cf78a8530a49b76af9e47b3f8e11266 Mon Sep 17 00:00:00 2001 From: idky137 Date: Wed, 27 May 2026 13:54:11 +0100 Subject: [PATCH 6/8] fix txout set accumulator for spends from non-standard script types --- .../db/v1/transparent_address_history.rs | 764 +++++++++++------- .../src/chain_index/types/db/metadata.rs | 89 ++ 2 files changed, 581 insertions(+), 272 deletions(-) diff --git a/packages/zaino-state/src/chain_index/finalised_state/db/v1/transparent_address_history.rs b/packages/zaino-state/src/chain_index/finalised_state/db/v1/transparent_address_history.rs index 073ff4765..0874baee0 100644 --- a/packages/zaino-state/src/chain_index/finalised_state/db/v1/transparent_address_history.rs +++ b/packages/zaino-state/src/chain_index/finalised_state/db/v1/transparent_address_history.rs @@ -19,6 +19,251 @@ enum AccumulatorDirection { Reverse, } +/// Applies a list of UTXO entries to the multiset commitment fields of the accumulator. +/// +/// For each entry the digest is XORed into `hash_serialized` (XOR is self-inverse, so the same +/// call site works for both add and remove). The integer fields `total_zatoshis` and +/// `bytes_serialized` move in the direction selected by `adding`. +fn apply_tx_out_set_entries_delta( + accumulator: &mut FinalisedTxOutSetInfoAccumulator, + entries: &[(Outpoint, TxOutCompact)], + adding: bool, +) -> Result<(), FinalisedStateError> { + for (outpoint, out) in entries { + let digest = tx_out_set_entry_digest(outpoint, out); + for (dst, src) in accumulator.hash_serialized.iter_mut().zip(digest.iter()) { + *dst ^= *src; + } + + if adding { + accumulator.total_zatoshis = accumulator + .total_zatoshis + .checked_add(out.value()) + .ok_or_else(|| { + FinalisedStateError::Custom( + "txout-set accumulator total_zatoshis overflow".to_string(), + ) + })?; + accumulator.bytes_serialized = accumulator + .bytes_serialized + .checked_add(ZAINO_TXOUTSET_ENTRY_LEN) + .ok_or_else(|| { + FinalisedStateError::Custom( + "txout-set accumulator bytes_serialized overflow".to_string(), + ) + })?; + } else { + accumulator.total_zatoshis = accumulator + .total_zatoshis + .checked_sub(out.value()) + .ok_or_else(|| { + FinalisedStateError::Custom( + "txout-set accumulator total_zatoshis underflow".to_string(), + ) + })?; + accumulator.bytes_serialized = accumulator + .bytes_serialized + .checked_sub(ZAINO_TXOUTSET_ENTRY_LEN) + .ok_or_else(|| { + FinalisedStateError::Custom( + "txout-set accumulator bytes_serialized underflow".to_string(), + ) + })?; + } + } + Ok(()) +} + +/// Applies the in-block portion of the accumulator update. +/// +/// Handles both the bulk `transaction_outputs` delta and the per-tx 0↔>0 transition that +/// counts a same-block transaction as entering (apply) or leaving (reverse) the UTXO set. +/// The positional bound check (`spent_index >= created_count`) uses the *full* output +/// count via `spent_indices_by_tx`; the UTXO-set membership transition uses +/// `spendable_spent_count_by_tx` which excludes unspendable outputs. +fn apply_in_block_transitions( + accumulator: &mut FinalisedTxOutSetInfoAccumulator, + created_counts: &HashMap, + spendable_counts: &HashMap, + spent_indices_by_tx: &HashMap>, + spendable_spent_count_by_tx: &HashMap, + spent_total_outputs: u64, + direction: AccumulatorDirection, +) -> Result<(), FinalisedStateError> { + let created_total = spendable_counts + .values() + .try_fold(0u64, |total, output_count| { + total.checked_add(u64::from(*output_count)).ok_or_else(|| { + FinalisedStateError::Custom( + "txout-set accumulator created output count overflow".to_string(), + ) + }) + })?; + + accumulator.transaction_outputs = match direction { + AccumulatorDirection::Apply => accumulator + .transaction_outputs + .checked_add(created_total) + .and_then(|v| v.checked_sub(spent_total_outputs)), + AccumulatorDirection::Reverse => accumulator + .transaction_outputs + .checked_sub(created_total) + .and_then(|v| v.checked_add(spent_total_outputs)), + } + .ok_or_else(|| { + FinalisedStateError::Custom( + "txout-set accumulator transaction output count underflow or overflow".to_string(), + ) + })?; + + for (transaction_hash, created_count) in created_counts { + let spent_indices = spent_indices_by_tx.get(transaction_hash); + + if let Some(spent_indices) = spent_indices { + for spent_index in spent_indices { + if spent_index >= created_count { + return Err(FinalisedStateError::Custom(format!( + "txout-set accumulator cannot be calculated: transaction {transaction_hash:?} spends same-block output index {spent_index}, but the transaction only has {created_count} transparent outputs" + ))); + } + } + } + + let spent_count = + spendable_spent_count_by_tx.get(transaction_hash).copied().unwrap_or(0); + + let spendable_count = spendable_counts.get(transaction_hash).copied().unwrap_or(0); + + if spendable_count > spent_count { + accumulator.transactions = match direction { + AccumulatorDirection::Apply => accumulator.transactions.checked_add(1), + AccumulatorDirection::Reverse => accumulator.transactions.checked_sub(1), + } + .ok_or_else(|| { + FinalisedStateError::Custom( + "txout-set accumulator transaction count underflow or overflow".to_string(), + ) + })?; + } + } + + Ok(()) +} + +/// Applies the per-entry deltas to `hash_serialized`, `bytes_serialized` and +/// `total_zatoshis`. +/// +/// Both `created_entries` and `spent_entries` must already be filtered to exclude +/// unspendable outputs — they were never in the UTXO set. +fn apply_entry_deltas( + accumulator: &mut FinalisedTxOutSetInfoAccumulator, + created_entries: &[(Outpoint, TxOutCompact)], + spent_entries: &[(Outpoint, TxOutCompact)], + direction: AccumulatorDirection, +) -> Result<(), FinalisedStateError> { + let (created_adding, spent_adding) = match direction { + AccumulatorDirection::Apply => (true, false), + AccumulatorDirection::Reverse => (false, true), + }; + + apply_tx_out_set_entries_delta(accumulator, created_entries, created_adding)?; + apply_tx_out_set_entries_delta(accumulator, spent_entries, spent_adding)?; + + Ok(()) +} + +/// Builds the per-transaction output count maps used by the accumulator helpers. +/// +/// Returns `(total_count_by_tx, spendable_count_by_tx)`: +/// - `total_count_by_tx` counts every transparent output and is used for positional +/// consensus bound checks. +/// - `spendable_count_by_tx` excludes provably-unspendable outputs (see +/// [`is_unspendable_tx_out`]) and is what drives UTXO-set deltas. +#[allow(clippy::type_complexity)] +fn index_created_outputs( + transactions: &[(TransactionHash, Option)], +) -> Result<(HashMap, HashMap), FinalisedStateError> { + let mut total_by_tx: HashMap = + HashMap::with_capacity(transactions.len()); + let mut spendable_by_tx: HashMap = + HashMap::with_capacity(transactions.len()); + + for (transaction_hash, transparent_transaction) in transactions { + let (total, spendable) = transparent_transaction + .as_ref() + .map(|tx| { + let total = tx.outputs().len(); + let spendable = tx + .outputs() + .iter() + .filter(|o| !is_unspendable_tx_out(o)) + .count(); + (total, spendable) + }) + .unwrap_or((0, 0)); + + let total = u32::try_from(total).map_err(|_| { + FinalisedStateError::Custom( + "txout-set accumulator cannot be calculated: transparent output count does not fit into u32" + .to_string(), + ) + })?; + let spendable = u32::try_from(spendable).map_err(|_| { + FinalisedStateError::Custom( + "txout-set accumulator cannot be calculated: spendable output count does not fit into u32" + .to_string(), + ) + })?; + + if total_by_tx.insert(*transaction_hash, total).is_some() { + return Err(FinalisedStateError::Custom(format!( + "txout-set accumulator cannot be calculated: duplicate transaction hash in block: {transaction_hash:?}" + ))); + } + spendable_by_tx.insert(*transaction_hash, spendable); + } + + Ok((total_by_tx, spendable_by_tx)) +} + +/// Groups a block's spent outpoints by the transaction they spend from. +/// +/// Returns `(spent_indices_by_tx, spent_outpoints_with_locations)`. The forward path +/// projects out just the outpoints; the reverse path needs the locations to verify the +/// spent index points to this block. +#[allow(clippy::type_complexity)] +fn index_spent_outpoints( + spent_map: &HashMap, +) -> Result< + ( + HashMap>, + Vec<(Outpoint, TxLocation)>, + ), + FinalisedStateError, +> { + let mut by_tx: HashMap> = HashMap::new(); + let mut outpoints = Vec::with_capacity(spent_map.len()); + + for (outpoint, tx_location) in spent_map.iter() { + let previous_transaction_hash = TransactionHash::from(*outpoint.prev_txid()); + + let inserted = by_tx + .entry(previous_transaction_hash) + .or_default() + .insert(outpoint.prev_index()); + + if !inserted { + return Err(FinalisedStateError::Custom(format!( + "txout-set accumulator cannot be calculated: duplicate transparent spend for outpoint {outpoint:?}" + ))); + } + + outpoints.push((*outpoint, *tx_location)); + } + + Ok((by_tx, outpoints)) +} + /// [`TransparentHistExt`] capability implementation for [`DbV1`]. /// /// Provides address history queries built over the LMDB `DUP_SORT`/`DUP_FIXED` address-history @@ -1072,61 +1317,6 @@ impl DbV1 { None } - /// Applies a list of UTXO entries to the multiset commitment fields of the accumulator. - /// - /// For each entry the digest is XORed into `hash_serialized` (XOR is self-inverse, so the same - /// call site works for both add and remove). The integer fields `total_zatoshis` and - /// `bytes_serialized` move in the direction selected by `adding`. - fn apply_tx_out_set_entries_delta( - accumulator: &mut FinalisedTxOutSetInfoAccumulator, - entries: &[(Outpoint, TxOutCompact)], - adding: bool, - ) -> Result<(), FinalisedStateError> { - for (outpoint, out) in entries { - let digest = tx_out_set_entry_digest(outpoint, out); - for (dst, src) in accumulator.hash_serialized.iter_mut().zip(digest.iter()) { - *dst ^= *src; - } - - if adding { - accumulator.total_zatoshis = accumulator - .total_zatoshis - .checked_add(out.value()) - .ok_or_else(|| { - FinalisedStateError::Custom( - "txout-set accumulator total_zatoshis overflow".to_string(), - ) - })?; - accumulator.bytes_serialized = accumulator - .bytes_serialized - .checked_add(ZAINO_TXOUTSET_ENTRY_LEN) - .ok_or_else(|| { - FinalisedStateError::Custom( - "txout-set accumulator bytes_serialized overflow".to_string(), - ) - })?; - } else { - accumulator.total_zatoshis = accumulator - .total_zatoshis - .checked_sub(out.value()) - .ok_or_else(|| { - FinalisedStateError::Custom( - "txout-set accumulator total_zatoshis underflow".to_string(), - ) - })?; - accumulator.bytes_serialized = accumulator - .bytes_serialized - .checked_sub(ZAINO_TXOUTSET_ENTRY_LEN) - .ok_or_else(|| { - FinalisedStateError::Custom( - "txout-set accumulator bytes_serialized underflow".to_string(), - ) - })?; - } - } - Ok(()) - } - /// Resolves each spent outpoint to its previous [`TxOutCompact`]. /// /// Same-block spends are resolved from the in-block `transactions` slice via the @@ -1166,179 +1356,6 @@ impl DbV1 { Ok(resolved) } - /// Builds the per-transaction output count maps used by the accumulator helpers. - /// - /// Returns `(total_count_by_tx, spendable_count_by_tx)`: - /// - `total_count_by_tx` counts every transparent output and is used for positional - /// consensus bound checks. - /// - `spendable_count_by_tx` excludes provably-unspendable outputs (see - /// [`is_unspendable_tx_out`]) and is what drives UTXO-set deltas. - #[allow(clippy::type_complexity)] - fn index_created_outputs( - transactions: &[(TransactionHash, Option)], - ) -> Result<(HashMap, HashMap), FinalisedStateError> - { - let mut total_by_tx: HashMap = - HashMap::with_capacity(transactions.len()); - let mut spendable_by_tx: HashMap = - HashMap::with_capacity(transactions.len()); - - for (transaction_hash, transparent_transaction) in transactions { - let (total, spendable) = transparent_transaction - .as_ref() - .map(|tx| { - let total = tx.outputs().len(); - let spendable = tx - .outputs() - .iter() - .filter(|o| !is_unspendable_tx_out(o)) - .count(); - (total, spendable) - }) - .unwrap_or((0, 0)); - - let total = u32::try_from(total).map_err(|_| { - FinalisedStateError::Custom( - "txout-set accumulator cannot be calculated: transparent output count does not fit into u32" - .to_string(), - ) - })?; - let spendable = u32::try_from(spendable).map_err(|_| { - FinalisedStateError::Custom( - "txout-set accumulator cannot be calculated: spendable output count does not fit into u32" - .to_string(), - ) - })?; - - // Duplicate txids would make the transaction-level accumulator ambiguous. - if total_by_tx.insert(*transaction_hash, total).is_some() { - return Err(FinalisedStateError::Custom(format!( - "txout-set accumulator cannot be calculated: duplicate transaction hash in block: {transaction_hash:?}" - ))); - } - spendable_by_tx.insert(*transaction_hash, spendable); - } - - Ok((total_by_tx, spendable_by_tx)) - } - - /// Groups a block's spent outpoints by the transaction they spend from. - /// - /// Returns `(spent_indices_by_tx, spent_outpoints_with_locations)`. The forward path - /// projects out just the outpoints; the reverse path needs the locations to verify the - /// spent index points to this block. - #[allow(clippy::type_complexity)] - fn index_spent_outpoints( - spent_map: &HashMap, - ) -> Result< - ( - HashMap>, - Vec<(Outpoint, TxLocation)>, - ), - FinalisedStateError, - > { - let mut by_tx: HashMap> = HashMap::new(); - let mut outpoints = Vec::with_capacity(spent_map.len()); - - for (outpoint, tx_location) in spent_map.iter() { - let previous_transaction_hash = TransactionHash::from(*outpoint.prev_txid()); - - let inserted = by_tx - .entry(previous_transaction_hash) - .or_default() - .insert(outpoint.prev_index()); - - if !inserted { - return Err(FinalisedStateError::Custom(format!( - "txout-set accumulator cannot be calculated: duplicate transparent spend for outpoint {outpoint:?}" - ))); - } - - outpoints.push((*outpoint, *tx_location)); - } - - Ok((by_tx, outpoints)) - } - - /// Applies the in-block portion of the accumulator update. - /// - /// Handles both the bulk `transaction_outputs` delta and the per-tx 0↔>0 transition that - /// counts a same-block transaction as entering (apply) or leaving (reverse) the UTXO set. - /// The positional bound check (`spent_index >= created_count`) uses the *full* output - /// count; the transition compares against the *spendable* count. - fn apply_in_block_transitions( - accumulator: &mut FinalisedTxOutSetInfoAccumulator, - created_counts: &HashMap, - spendable_counts: &HashMap, - spent_indices_by_tx: &HashMap>, - spent_total_outputs: u64, - direction: AccumulatorDirection, - ) -> Result<(), FinalisedStateError> { - let created_total = spendable_counts - .values() - .try_fold(0u64, |total, output_count| { - total.checked_add(u64::from(*output_count)).ok_or_else(|| { - FinalisedStateError::Custom( - "txout-set accumulator created output count overflow".to_string(), - ) - }) - })?; - - accumulator.transaction_outputs = match direction { - AccumulatorDirection::Apply => accumulator - .transaction_outputs - .checked_add(created_total) - .and_then(|v| v.checked_sub(spent_total_outputs)), - AccumulatorDirection::Reverse => accumulator - .transaction_outputs - .checked_sub(created_total) - .and_then(|v| v.checked_add(spent_total_outputs)), - } - .ok_or_else(|| { - FinalisedStateError::Custom( - "txout-set accumulator transaction output count underflow or overflow".to_string(), - ) - })?; - - for (transaction_hash, created_count) in created_counts { - let spent_indices = spent_indices_by_tx.get(transaction_hash); - - if let Some(spent_indices) = spent_indices { - for spent_index in spent_indices { - if spent_index >= created_count { - return Err(FinalisedStateError::Custom(format!( - "txout-set accumulator cannot be calculated: transaction {transaction_hash:?} spends same-block output index {spent_index}, but the transaction only has {created_count} transparent outputs" - ))); - } - } - } - - let spent_count = spent_indices.map(|s| s.len()).unwrap_or(0); - let spent_count = u32::try_from(spent_count).map_err(|_| { - FinalisedStateError::Custom( - "txout-set accumulator same-block spent output count does not fit into u32" - .to_string(), - ) - })?; - - let spendable_count = spendable_counts.get(transaction_hash).copied().unwrap_or(0); - - if spendable_count > spent_count { - accumulator.transactions = match direction { - AccumulatorDirection::Apply => accumulator.transactions.checked_add(1), - AccumulatorDirection::Reverse => accumulator.transactions.checked_sub(1), - } - .ok_or_else(|| { - FinalisedStateError::Custom( - "txout-set accumulator transaction count underflow or overflow".to_string(), - ) - })?; - } - } - - Ok(()) - } - /// Applies the prior-block portion of the accumulator update. /// /// For every transaction spent from by this block that was *not* created in this block, @@ -1430,20 +1447,27 @@ impl DbV1 { Ok(()) } - /// Applies the per-entry deltas to `hash_serialized`, `bytes_serialized` and - /// `total_zatoshis`. + /// Resolves and filters the created and spent entry lists for accumulator updates. + /// + /// Created entries are collected from the block's transactions, excluding unspendable + /// outputs. Spent entries are resolved (same-block from `transactions`, prior-block from + /// the database) and likewise filtered to exclude unspendable outputs. /// - /// Created outputs come from the paired `transactions` slice; spent prev outputs are - /// resolved via [`Self::resolve_spent_outpoints_for_set_info`] (same-block from the slice, - /// prior-block from the database). NonStandard outputs are skipped on both sides — they - /// were never in the UTXO set. - async fn apply_entry_deltas( + /// Returns `(created_entries, spent_entries, spendable_spent_count_by_tx)`. + /// `spendable_spent_count_by_tx` counts only spendable same-block spends per source tx. + #[allow(clippy::type_complexity)] + fn build_entry_data( &self, - accumulator: &mut FinalisedTxOutSetInfoAccumulator, transactions: &[(TransactionHash, Option)], spent_map: &HashMap, - direction: AccumulatorDirection, - ) -> Result<(), FinalisedStateError> { + ) -> Result< + ( + Vec<(Outpoint, TxOutCompact)>, + Vec<(Outpoint, TxOutCompact)>, + HashMap, + ), + FinalisedStateError, + > { let mut created_entries: Vec<(Outpoint, TxOutCompact)> = Vec::new(); let mut txid_to_block_index: HashMap = HashMap::with_capacity(transactions.len()); @@ -1466,21 +1490,25 @@ impl DbV1 { } } - let spent_entries = self.resolve_spent_outpoints_for_set_info( + let resolved = self.resolve_spent_outpoints_for_set_info( spent_map, &txid_to_block_index, transactions, )?; - let (created_adding, spent_adding) = match direction { - AccumulatorDirection::Apply => (true, false), - AccumulatorDirection::Reverse => (false, true), - }; + let mut spent_entries = Vec::with_capacity(resolved.len()); + let mut spendable_spent_count_by_tx: HashMap = HashMap::new(); - Self::apply_tx_out_set_entries_delta(accumulator, &created_entries, created_adding)?; - Self::apply_tx_out_set_entries_delta(accumulator, &spent_entries, spent_adding)?; + for (outpoint, out) in resolved { + if is_unspendable_tx_out(&out) { + continue; + } + let prev_txid = TransactionHash::from(*outpoint.prev_txid()); + *spendable_spent_count_by_tx.entry(prev_txid).or_default() += 1; + spent_entries.push((outpoint, out)); + } - Ok(()) + Ok((created_entries, spent_entries, spendable_spent_count_by_tx)) } /// Calculates the finalised txout-set accumulator after applying the block currently being written. @@ -1520,13 +1548,8 @@ impl DbV1 { Err(error) => return Err(error), }; - let (created_counts, spendable_counts) = Self::index_created_outputs(transactions)?; - let (spent_indices_by_tx, spent_outpoints) = Self::index_spent_outpoints(spent_map)?; - let spent_total_outputs = u64::try_from(spent_outpoints.len()).map_err(|_| { - FinalisedStateError::Custom( - "txout-set accumulator spent output count does not fit into u64".to_string(), - ) - })?; + let (created_counts, spendable_counts) = index_created_outputs(transactions)?; + let (spent_indices_by_tx, spent_outpoints) = index_spent_outpoints(spent_map)?; // Forward-direction validation: outpoints spent by this block must not already be // spent in finalised state (same-block spends are not in the finalised spent table @@ -1549,11 +1572,21 @@ impl DbV1 { } } - Self::apply_in_block_transitions( + let (created_entries, spent_entries, spendable_spent_count_by_tx) = + self.build_entry_data(transactions, spent_map)?; + + let spent_total_outputs = u64::try_from(spent_entries.len()).map_err(|_| { + FinalisedStateError::Custom( + "txout-set accumulator spent output count does not fit into u64".to_string(), + ) + })?; + + apply_in_block_transitions( &mut accumulator, &created_counts, &spendable_counts, &spent_indices_by_tx, + &spendable_spent_count_by_tx, spent_total_outputs, AccumulatorDirection::Apply, )?; @@ -1564,13 +1597,12 @@ impl DbV1 { AccumulatorDirection::Apply, ) .await?; - self.apply_entry_deltas( + apply_entry_deltas( &mut accumulator, - transactions, - spent_map, + &created_entries, + &spent_entries, AccumulatorDirection::Apply, - ) - .await?; + )?; Ok(accumulator) } @@ -1597,13 +1629,8 @@ impl DbV1 { Err(error) => return Err(error), }; - let (created_counts, spendable_counts) = Self::index_created_outputs(transactions)?; - let (spent_indices_by_tx, spent_outpoints) = Self::index_spent_outpoints(spent_map)?; - let spent_total_outputs = u64::try_from(spent_outpoints.len()).map_err(|_| { - FinalisedStateError::Custom( - "txout-set accumulator spent output count does not fit into u64".to_string(), - ) - })?; + let (created_counts, spendable_counts) = index_created_outputs(transactions)?; + let (spent_indices_by_tx, spent_outpoints) = index_spent_outpoints(spent_map)?; // Reverse-direction validation: every spent outpoint from this block must be in the // finalised spent index and must point to this block's TxLocation. @@ -1627,11 +1654,21 @@ impl DbV1 { } } - Self::apply_in_block_transitions( + let (created_entries, spent_entries, spendable_spent_count_by_tx) = + self.build_entry_data(transactions, spent_map)?; + + let spent_total_outputs = u64::try_from(spent_entries.len()).map_err(|_| { + FinalisedStateError::Custom( + "txout-set accumulator spent output count does not fit into u64".to_string(), + ) + })?; + + apply_in_block_transitions( &mut accumulator, &created_counts, &spendable_counts, &spent_indices_by_tx, + &spendable_spent_count_by_tx, spent_total_outputs, AccumulatorDirection::Reverse, )?; @@ -1642,14 +1679,197 @@ impl DbV1 { AccumulatorDirection::Reverse, ) .await?; - self.apply_entry_deltas( + apply_entry_deltas( &mut accumulator, - transactions, - spent_map, + &created_entries, + &spent_entries, AccumulatorDirection::Reverse, - ) - .await?; + )?; Ok(accumulator) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::chain_index::types::db::metadata::{ + FinalisedTxOutSetInfoAccumulator, ZAINO_TXOUTSET_ENTRY_LEN, + }; + + fn p2pkh_out(value: u64) -> TxOutCompact { + TxOutCompact::new(value, [0x11; 20], 0).expect("P2PKH script_type should be valid") + } + + fn outpoint(txid_byte: u8, index: u32) -> Outpoint { + Outpoint::new([txid_byte; 32], index) + } + + #[test] + fn entries_delta_add_then_remove_roundtrips() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + let entries = vec![ + (outpoint(0x01, 0), p2pkh_out(100)), + (outpoint(0x02, 1), p2pkh_out(200)), + ]; + + apply_tx_out_set_entries_delta(&mut acc, &entries, true) + .expect("add should succeed"); + + assert_eq!(acc.total_zatoshis, 300); + assert_eq!(acc.bytes_serialized, 2 * ZAINO_TXOUTSET_ENTRY_LEN); + + apply_tx_out_set_entries_delta(&mut acc, &entries, false) + .expect("remove should succeed"); + + assert_eq!(acc, FinalisedTxOutSetInfoAccumulator::empty()); + } + + #[test] + fn entries_delta_remove_on_empty_returns_underflow_error() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + let entries = vec![(outpoint(0xAA, 0), p2pkh_out(500))]; + + let err = apply_tx_out_set_entries_delta(&mut acc, &entries, false); + + assert!(err.is_err()); + let msg = err.unwrap_err().to_string(); + assert!(msg.contains("underflow"), "expected underflow, got: {msg}"); + } + + #[test] + fn entries_delta_ignores_empty_slice() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + acc.total_zatoshis = 999; + acc.bytes_serialized = 65; + acc.transaction_outputs = 1; + + let snapshot = acc; + apply_tx_out_set_entries_delta(&mut acc, &[], true) + .expect("empty add should succeed"); + assert_eq!(acc, snapshot); + + apply_tx_out_set_entries_delta(&mut acc, &[], false) + .expect("empty remove should succeed"); + assert_eq!(acc, snapshot); + } + + #[test] + fn in_block_transitions_spendable_only() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + let tx_hash = TransactionHash([0xAB; 32]); + + let created_counts = HashMap::from([(tx_hash, 3)]); + let spendable_counts = HashMap::from([(tx_hash, 2)]); + let spent_indices_by_tx = HashMap::from([(tx_hash, HashSet::from([0]))]); + let spendable_spent_count_by_tx = HashMap::from([(tx_hash, 1)]); + + apply_in_block_transitions( + &mut acc, + &created_counts, + &spendable_counts, + &spent_indices_by_tx, + &spendable_spent_count_by_tx, + 1, + AccumulatorDirection::Apply, + ) + .expect("apply should succeed"); + + assert_eq!(acc.transaction_outputs, 1, "2 created - 1 spent = 1"); + assert_eq!(acc.transactions, 1, "tx enters UTXO set: 2 spendable > 1 spent"); + } + + #[test] + fn in_block_transitions_unspendable_spend_does_not_inflate_count() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + let tx_hash = TransactionHash([0xCC; 32]); + + // Tx has 2 total outputs, 1 spendable (P2PKH at idx 0) + 1 unspendable (NonStandard at idx 1). + // The unspendable output is spent in the same block, but after filtering it's excluded. + let created_counts = HashMap::from([(tx_hash, 2)]); + let spendable_counts = HashMap::from([(tx_hash, 1)]); + // Full indices include the unspendable spend for positional check. + let spent_indices_by_tx = HashMap::from([(tx_hash, HashSet::from([1]))]); + // After filtering: no spendable outputs were spent. + let spendable_spent_count_by_tx = HashMap::new(); + + apply_in_block_transitions( + &mut acc, + &created_counts, + &spendable_counts, + &spent_indices_by_tx, + &spendable_spent_count_by_tx, + 0, + AccumulatorDirection::Apply, + ) + .expect("apply should succeed"); + + assert_eq!(acc.transaction_outputs, 1, "1 spendable created - 0 spendable spent"); + assert_eq!(acc.transactions, 1, "tx enters UTXO set: 1 spendable > 0 spent"); + } + + #[test] + fn in_block_transitions_all_spendable_spent_same_block() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + let tx_hash = TransactionHash([0xDD; 32]); + + let created_counts = HashMap::from([(tx_hash, 2)]); + let spendable_counts = HashMap::from([(tx_hash, 2)]); + let spent_indices_by_tx = HashMap::from([(tx_hash, HashSet::from([0, 1]))]); + let spendable_spent_count_by_tx = HashMap::from([(tx_hash, 2)]); + + apply_in_block_transitions( + &mut acc, + &created_counts, + &spendable_counts, + &spent_indices_by_tx, + &spendable_spent_count_by_tx, + 2, + AccumulatorDirection::Apply, + ) + .expect("apply should succeed"); + + assert_eq!(acc.transaction_outputs, 0, "2 created - 2 spent = 0"); + assert_eq!(acc.transactions, 0, "tx never enters UTXO set: 2 == 2"); + } + + #[test] + fn in_block_transitions_reverse_direction() { + let tx_hash = TransactionHash([0xEE; 32]); + + let created_counts = HashMap::from([(tx_hash, 2)]); + let spendable_counts = HashMap::from([(tx_hash, 2)]); + let spent_indices_by_tx = HashMap::new(); + let spendable_spent_count_by_tx = HashMap::new(); + + // Simulate state after writing a block that created 2 spendable outputs. + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + apply_in_block_transitions( + &mut acc, + &created_counts, + &spendable_counts, + &spent_indices_by_tx, + &spendable_spent_count_by_tx, + 0, + AccumulatorDirection::Apply, + ) + .expect("forward apply should succeed"); + + assert_eq!(acc.transaction_outputs, 2); + assert_eq!(acc.transactions, 1); + + // Reverse should return to empty. + apply_in_block_transitions( + &mut acc, + &created_counts, + &spendable_counts, + &spent_indices_by_tx, + &spendable_spent_count_by_tx, + 0, + AccumulatorDirection::Reverse, + ) + .expect("reverse should succeed"); + + assert_eq!(acc, FinalisedTxOutSetInfoAccumulator::empty()); + } +} diff --git a/packages/zaino-state/src/chain_index/types/db/metadata.rs b/packages/zaino-state/src/chain_index/types/db/metadata.rs index 28956d55f..4fcc1b7cb 100644 --- a/packages/zaino-state/src/chain_index/types/db/metadata.rs +++ b/packages/zaino-state/src/chain_index/types/db/metadata.rs @@ -371,4 +371,93 @@ mod tests { // We just sanity-check the digest is not all zeros. assert_ne!(a, [0u8; 32]); } + + #[test] + fn is_unspendable_filters_non_standard() { + let out = TxOutCompact::new(100, [0x00; 20], 0xFF) + .expect("script_type 0xFF (NonStandard) should be valid"); + assert!(is_unspendable_tx_out(&out)); + } + + #[test] + fn is_unspendable_allows_p2pkh() { + let out = + TxOutCompact::new(100, [0x00; 20], 0).expect("script_type 0 (P2PKH) should be valid"); + assert!(!is_unspendable_tx_out(&out)); + } + + #[test] + fn is_unspendable_allows_p2sh() { + let out = + TxOutCompact::new(100, [0x00; 20], 1).expect("script_type 1 (P2SH) should be valid"); + assert!(!is_unspendable_tx_out(&out)); + } + + #[test] + fn apply_added_then_removed_output_returns_to_empty() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + let outpoint = Outpoint::new([0xAA; 32], 0); + let out = TxOutCompact::new(50_000, [0x11; 20], 0) + .expect("script_type 0 (P2PKH) should be valid"); + + acc.apply_added_output(&outpoint, &out) + .expect("add should succeed"); + + assert_ne!(acc, FinalisedTxOutSetInfoAccumulator::empty()); + assert_eq!(acc.total_zatoshis, 50_000); + assert_eq!(acc.transaction_outputs, 1); + assert_eq!(acc.bytes_serialized, ZAINO_TXOUTSET_ENTRY_LEN); + + acc.apply_removed_output(&outpoint, &out) + .expect("remove should succeed"); + + assert_eq!(acc, FinalisedTxOutSetInfoAccumulator::empty()); + } + + #[test] + fn apply_removed_output_on_empty_underflows() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + let outpoint = Outpoint::new([0xBB; 32], 0); + let out = TxOutCompact::new(1_000, [0x22; 20], 0) + .expect("script_type 0 (P2PKH) should be valid"); + + let err = acc + .apply_removed_output(&outpoint, &out) + .expect_err("remove on empty should underflow"); + + assert_eq!( + err, + AccumulatorDeltaError::Underflow("transaction_outputs") + ); + } + + #[test] + fn apply_added_output_accumulates_values() { + let mut acc = FinalisedTxOutSetInfoAccumulator::empty(); + + let outpoint_a = Outpoint::new([0x01; 32], 0); + let out_a = TxOutCompact::new(100, [0x11; 20], 0) + .expect("script_type 0 (P2PKH) should be valid"); + + let outpoint_b = Outpoint::new([0x02; 32], 1); + let out_b = TxOutCompact::new(200, [0x22; 20], 1) + .expect("script_type 1 (P2SH) should be valid"); + + acc.apply_added_output(&outpoint_a, &out_a) + .expect("add a should succeed"); + acc.apply_added_output(&outpoint_b, &out_b) + .expect("add b should succeed"); + + assert_eq!(acc.total_zatoshis, 300); + assert_eq!(acc.transaction_outputs, 2); + assert_eq!(acc.bytes_serialized, 2 * ZAINO_TXOUTSET_ENTRY_LEN); + + let digest_a = tx_out_set_entry_digest(&outpoint_a, &out_a); + let digest_b = tx_out_set_entry_digest(&outpoint_b, &out_b); + let mut expected_hash = [0u8; 32]; + for i in 0..32 { + expected_hash[i] = digest_a[i] ^ digest_b[i]; + } + assert_eq!(acc.hash_serialized, expected_hash); + } } From 0333a69249210744f0fd931898990b28c662cc41 Mon Sep 17 00:00:00 2001 From: Francisco Gindre Date: Wed, 27 May 2026 14:49:41 -0300 Subject: [PATCH 7/8] PR Suggestion --- packages/zaino-state/src/chain_index.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index c828d3274..1c540ac00 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -1053,6 +1053,7 @@ async fn compact_block_from_source( "orchard commitment tree size overflow", )) })?, + // TODO: Define an empty value https://github.com/zingolabs/zaino/issues/1158 ChainWork::from_u256(0.into()), network, ); From 8e391217f802b12dc9e9bb1d1c181e46cdd65475 Mon Sep 17 00:00:00 2001 From: Francisco Gindre Date: Wed, 27 May 2026 14:51:09 -0300 Subject: [PATCH 8/8] Fix compiler error --- packages/zaino-state/src/chain_index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zaino-state/src/chain_index.rs b/packages/zaino-state/src/chain_index.rs index 1c540ac00..183d79486 100644 --- a/packages/zaino-state/src/chain_index.rs +++ b/packages/zaino-state/src/chain_index.rs @@ -20,7 +20,7 @@ use crate::chain_index::types::{BestChainLocation, NonBestChainLocation}; use crate::error::{ChainIndexError, ChainIndexErrorKind, FinalisedStateError}; use crate::status::Status; use crate::{ - ChainWork, CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError, + ChainWork, CompactBlockStream, NamedAtomicStatus, NonFinalizedState, StatusType, SyncError, TxOutCompact, }; use crate::{IndexedBlock, Outpoint, TransactionHash}; use std::collections::HashSet;