diff --git a/zebra-chain/src/transaction/arbitrary.rs b/zebra-chain/src/transaction/arbitrary.rs index 1761287c..c9cf3686 100644 --- a/zebra-chain/src/transaction/arbitrary.rs +++ b/zebra-chain/src/transaction/arbitrary.rs @@ -293,7 +293,7 @@ impl Transaction { for input in self.inputs() { input_chain_value_pools = input_chain_value_pools - .update_with_transparent_input(input, outputs) + .add_transparent_input(input, outputs) .expect("find_valid_utxo_for_spend only spends unspent transparent outputs"); } @@ -304,7 +304,7 @@ impl Transaction { // so at least one of the values in each JoinSplit is zero for input in self.input_values_from_sprout_mut() { match input_chain_value_pools - .update_with_chain_value_pool_change(ValueBalance::from_sprout_amount(input.neg())) + .add_chain_value_pool_change(ValueBalance::from_sprout_amount(input.neg())) { Ok(new_chain_pools) => input_chain_value_pools = new_chain_pools, // set the invalid input value to zero @@ -316,21 +316,17 @@ impl Transaction { let sapling_input = self.sapling_value_balance().constrain::(); if let Ok(sapling_input) = sapling_input { - if sapling_input != ValueBalance::zero() { - match input_chain_value_pools.update_with_chain_value_pool_change(-sapling_input) { - Ok(new_chain_pools) => input_chain_value_pools = new_chain_pools, - Err(_) => *self.sapling_value_balance_mut().unwrap() = Amount::zero(), - } + match input_chain_value_pools.add_chain_value_pool_change(-sapling_input) { + Ok(new_chain_pools) => input_chain_value_pools = new_chain_pools, + Err(_) => *self.sapling_value_balance_mut().unwrap() = Amount::zero(), } } let orchard_input = self.orchard_value_balance().constrain::(); if let Ok(orchard_input) = orchard_input { - if orchard_input != ValueBalance::zero() { - match input_chain_value_pools.update_with_chain_value_pool_change(-orchard_input) { - Ok(new_chain_pools) => input_chain_value_pools = new_chain_pools, - Err(_) => *self.orchard_value_balance_mut().unwrap() = Amount::zero(), - } + match input_chain_value_pools.add_chain_value_pool_change(-orchard_input) { + Ok(new_chain_pools) => input_chain_value_pools = new_chain_pools, + Err(_) => *self.orchard_value_balance_mut().unwrap() = Amount::zero(), } } @@ -344,7 +340,7 @@ impl Transaction { .neg(); let chain_value_pools = chain_value_pools - .update_with_transaction(self, outputs) + .add_transaction(self, outputs) .unwrap_or_else(|err| { panic!( "unexpected chain value pool error: {:?}, \n\ diff --git a/zebra-chain/src/value_balance.rs b/zebra-chain/src/value_balance.rs index d51ff020..7e05808b 100644 --- a/zebra-chain/src/value_balance.rs +++ b/zebra-chain/src/value_balance.rs @@ -160,7 +160,7 @@ impl ValueBalance { } impl ValueBalance { - /// Returns this value balance, updated with the chain value pool change from `block`. + /// Return the sum of the chain value pool change from `block`, and this value balance. /// /// `utxos` must contain the [`Utxo`]s of every input in this block, /// including UTXOs created by earlier transactions in this block. @@ -181,17 +181,17 @@ impl ValueBalance { /// value pool. /// /// See [`Block::chain_value_pool_change`] for details. - pub fn update_with_block( + pub fn add_block( self, block: impl Borrow, utxos: &HashMap, ) -> Result, ValueBalanceError> { let chain_value_pool_change = block.borrow().chain_value_pool_change(utxos)?; - self.update_with_chain_value_pool_change(chain_value_pool_change) + self.add_chain_value_pool_change(chain_value_pool_change) } - /// Returns this value balance, updated with the chain value pool change from `transaction`. + /// Return the sum of the chain value pool change from `transaction`, and this value balance. /// /// `outputs` must contain the [`Output`]s of every input in this transaction, /// including UTXOs created by earlier transactions in its block. @@ -202,7 +202,7 @@ impl ValueBalance { /// See [`Block::chain_value_pool_change`] and [`Transaction::value_balance`] /// for details. #[cfg(any(test, feature = "proptest-impl"))] - pub fn update_with_transaction( + pub fn add_transaction( self, transaction: impl Borrow, utxos: &HashMap, @@ -216,10 +216,10 @@ impl ValueBalance { .value_balance_from_outputs(utxos)? .neg(); - self.update_with_chain_value_pool_change(chain_value_pool_change) + self.add_chain_value_pool_change(chain_value_pool_change) } - /// Returns this value balance, updated with the chain value pool change from `input`. + /// Return the sum of the chain value pool change from `input`, and this value balance. /// /// `outputs` must contain the [`Output`] spent by `input`, /// (including UTXOs created by earlier transactions in its block). @@ -230,7 +230,7 @@ impl ValueBalance { /// See [`Block::chain_value_pool_change`] and [`Transaction::value_balance`] /// for details. #[cfg(any(test, feature = "proptest-impl"))] - pub fn update_with_transparent_input( + pub fn add_transparent_input( self, input: impl Borrow, utxos: &HashMap, @@ -243,16 +243,16 @@ impl ValueBalance { let transparent_value_pool_change = ValueBalance::from_transparent_amount(transparent_value_pool_change); - self.update_with_chain_value_pool_change(transparent_value_pool_change) + self.add_chain_value_pool_change(transparent_value_pool_change) } - /// Returns this value balance, updated with a chain value pool change. + /// Return the sum of the chain value pool change, and this value balance. /// /// Note: the chain value pool has the opposite sign to the transaction /// value pool. /// - /// See `update_with_block` for details. - pub fn update_with_chain_value_pool_change( + /// See `add_block` for details. + pub fn add_chain_value_pool_change( self, chain_value_pool_change: ValueBalance, ) -> Result, ValueBalanceError> { diff --git a/zebra-state/src/arbitrary.rs b/zebra-state/src/arbitrary.rs index cc249495..d48c9aa2 100644 --- a/zebra-state/src/arbitrary.rs +++ b/zebra-state/src/arbitrary.rs @@ -1,8 +1,14 @@ use std::sync::Arc; -use zebra_chain::{block::Block, transparent}; +use zebra_chain::{ + amount::{Amount, NegativeAllowed}, + block::{self, Block}, + transaction::Transaction, + transparent, + value_balance::ValueBalance, +}; -use crate::PreparedBlock; +use crate::{request::ContextuallyValidBlock, PreparedBlock}; /// Mocks computation done during semantic validation pub trait Prepare { @@ -26,3 +32,98 @@ impl Prepare for Arc { } } } + +impl PreparedBlock { + /// Returns a [`ContextuallyValidBlock`] created from this block, + /// with fake zero-valued spent UTXOs. + /// + /// Only for use in tests. + pub fn test_with_zero_spent_utxos(&self) -> ContextuallyValidBlock { + ContextuallyValidBlock::test_with_zero_spent_utxos(self) + } + + /// Returns a [`ContextuallyValidBlock`] created from this block, + /// using a fake chain value pool change. + /// + /// Only for use in tests. + pub fn test_with_chain_pool_change( + &self, + fake_chain_value_pool_change: ValueBalance, + ) -> ContextuallyValidBlock { + ContextuallyValidBlock::test_with_chain_pool_change(self, fake_chain_value_pool_change) + } + + /// Returns a [`ContextuallyValidBlock`] created from this block, + /// with no chain value pool change. + /// + /// Only for use in tests. + pub fn test_with_zero_chain_pool_change(&self) -> ContextuallyValidBlock { + ContextuallyValidBlock::test_with_zero_chain_pool_change(self) + } +} + +impl ContextuallyValidBlock { + /// Create a block that's ready for non-finalized [`Chain`] contextual validation, + /// using a [`PreparedBlock`] and fake zero-valued spent UTXOs. + /// + /// Only for use in tests. + pub fn test_with_zero_spent_utxos(block: impl Into) -> Self { + let block = block.into(); + + let zero_utxo = transparent::Utxo { + output: transparent::Output { + value: Amount::zero(), + lock_script: transparent::Script::new(&[]), + }, + height: block::Height(1), + from_coinbase: false, + }; + + let zero_spent_utxos = block + .block + .transactions + .iter() + .map(AsRef::as_ref) + .flat_map(Transaction::inputs) + .flat_map(transparent::Input::outpoint) + .map(|outpoint| (outpoint, zero_utxo.clone())) + .collect(); + + ContextuallyValidBlock::with_block_and_spent_utxos(block, zero_spent_utxos) + .expect("all UTXOs are provided with zero values") + } + + /// Create a [`ContextuallyValidBlock`] from a [`Block`] or [`PreparedBlock`], + /// using a fake chain value pool change. + /// + /// Only for use in tests. + pub fn test_with_chain_pool_change( + block: impl Into, + fake_chain_value_pool_change: ValueBalance, + ) -> Self { + let PreparedBlock { + block, + hash, + height, + new_outputs, + transaction_hashes, + } = block.into(); + + Self { + block, + hash, + height, + new_outputs: transparent::utxos_from_ordered_utxos(new_outputs), + transaction_hashes, + chain_value_pool_change: fake_chain_value_pool_change, + } + } + + /// Create a [`ContextuallyValidBlock`] from a [`Block`] or [`PreparedBlock`], + /// with no chain value pool change. + /// + /// Only for use in tests. + pub fn test_with_zero_chain_pool_change(block: impl Into) -> Self { + Self::test_with_chain_pool_change(block, ValueBalance::zero()) + } +} diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 43dbdd3e..c0fead87 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -4,8 +4,12 @@ use chrono::{DateTime, Utc}; use thiserror::Error; use zebra_chain::{ - amount, block, history_tree::HistoryTreeError, orchard, sapling, sprout, transparent, - value_balance::ValueBalanceError, work::difficulty::CompactDifficulty, + amount::{self, NegativeAllowed, NonNegative}, + block, + history_tree::HistoryTreeError, + orchard, sapling, sprout, transaction, transparent, + value_balance::{ValueBalance, ValueBalanceError}, + work::difficulty::CompactDifficulty, }; use crate::constants::MIN_TRANSPARENT_COINBASE_MATURITY; @@ -142,42 +146,72 @@ pub enum ValidateContextError { }, #[error( - "the remaining value in the transparent transaction value pool MUST be nonnegative: \ - {amount_error:?}, {height:?}, index in block: {tx_index_in_block:?}, \ - {transaction_hash:?}" + "the remaining value in the transparent transaction value pool MUST be nonnegative:\n\ + {amount_error:?},\n\ + {height:?}, index in block: {tx_index_in_block:?}, {transaction_hash:?}" )] #[non_exhaustive] NegativeRemainingTransactionValue { amount_error: amount::Error, height: block::Height, tx_index_in_block: usize, - transaction_hash: zebra_chain::transaction::Hash, + transaction_hash: transaction::Hash, }, #[error( - "error calculating the remaining value in the transaction value pool: \ - {amount_error:?}, {height:?}, index in block: {tx_index_in_block:?}, \ - {transaction_hash:?}" + "error calculating the remaining value in the transaction value pool:\n\ + {amount_error:?},\n\ + {height:?}, index in block: {tx_index_in_block:?}, {transaction_hash:?}" )] #[non_exhaustive] CalculateRemainingTransactionValue { amount_error: amount::Error, height: block::Height, tx_index_in_block: usize, - transaction_hash: zebra_chain::transaction::Hash, + transaction_hash: transaction::Hash, }, #[error( - "error calculating value balances for the remaining value in the transaction value pool: \ - {value_balance_error:?}, {height:?}, index in block: {tx_index_in_block:?}, \ - {transaction_hash:?}" + "error calculating value balances for the remaining value in the transaction value pool:\n\ + {value_balance_error:?},\n\ + {height:?}, index in block: {tx_index_in_block:?}, {transaction_hash:?}" )] #[non_exhaustive] CalculateTransactionValueBalances { value_balance_error: ValueBalanceError, height: block::Height, tx_index_in_block: usize, - transaction_hash: zebra_chain::transaction::Hash, + transaction_hash: transaction::Hash, + }, + + #[error( + "error calculating the block chain value pool change:\n\ + {value_balance_error:?},\n\ + {height:?}, {block_hash:?},\n\ + transactions: {transaction_count:?}, spent UTXOs: {spent_utxo_count:?}" + )] + #[non_exhaustive] + CalculateBlockChainValueChange { + value_balance_error: ValueBalanceError, + height: block::Height, + block_hash: block::Hash, + transaction_count: usize, + spent_utxo_count: usize, + }, + + #[error( + "error adding value balances to the chain value pool:\n\ + {value_balance_error:?},\n\ + {chain_value_pools:?},\n\ + {block_value_pool_change:?},\n\ + {height:?}" + )] + #[non_exhaustive] + AddValuePool { + value_balance_error: ValueBalanceError, + chain_value_pools: ValueBalance, + block_value_pool_change: ValueBalance, + height: Option, }, #[error("error in Sapling note commitment tree")] diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index c66e5eb2..062aded0 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -1,8 +1,11 @@ use std::{collections::HashMap, sync::Arc}; use zebra_chain::{ + amount::NegativeAllowed, block::{self, Block}, - transaction, transparent, + transaction, + transparent::{self, utxos_from_ordered_utxos}, + value_balance::{ValueBalance, ValueBalanceError}, }; // Allow *only* this unused import, so that rustdoc link resolution @@ -78,9 +81,6 @@ pub struct PreparedBlock { pub new_outputs: HashMap, /// A precomputed list of the hashes of the transactions in this block. pub transaction_hashes: Vec, - // TODO: add these parameters when we can compute anchors. - // sprout_anchor: sprout::tree::Root, - // sapling_anchor: sapling::tree::Root, } /// A contextually validated block, ready to be committed directly to the finalized state with @@ -88,12 +88,14 @@ pub struct PreparedBlock { /// /// Used by the state service and non-finalized [`Chain`]. #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ContextuallyValidBlock { +pub struct ContextuallyValidBlock { pub(crate) block: Arc, pub(crate) hash: block::Hash, pub(crate) height: block::Height, pub(crate) new_outputs: HashMap, pub(crate) transaction_hashes: Vec, + /// The sum of the chain value pool changes of all transactions in this block. + pub(crate) chain_value_pool_change: ValueBalance, } /// A finalized block, ready to be committed directly to the finalized state with @@ -111,6 +113,49 @@ pub struct FinalizedBlock { pub(crate) transaction_hashes: Vec, } +impl From<&PreparedBlock> for PreparedBlock { + fn from(prepared: &PreparedBlock) -> Self { + prepared.clone() + } +} + +impl ContextuallyValidBlock { + /// Create a block that's ready for non-finalized [`Chain`] contextual validation, + /// using a [`PreparedBlock`] and the UTXOs it spends. + /// + /// When combined, `prepared.new_outputs` and `spent_utxos` must contain + /// the [`Utxo`]s spent by every transparent input in this block, + /// including UTXOs created by earlier transactions in this block. + /// + /// Note: a [`ContextuallyValidBlock`] isn't actually contextually valid until + /// [`Chain::update_chain_state_with`] returns success. + pub fn with_block_and_spent_utxos( + prepared: PreparedBlock, + mut spent_utxos: HashMap, + ) -> Result { + let PreparedBlock { + block, + hash, + height, + new_outputs, + transaction_hashes, + } = prepared; + + // This is redundant for the non-finalized state, + // but useful to make some tests pass more easily. + spent_utxos.extend(utxos_from_ordered_utxos(new_outputs.clone())); + + Ok(Self { + block: block.clone(), + hash, + height, + new_outputs: transparent::utxos_from_ordered_utxos(new_outputs), + transaction_hashes, + chain_value_pool_change: block.chain_value_pool_change(&spent_utxos)?, + }) + } +} + // Doing precomputation in this From impl means that it will be done in // the *service caller*'s task, not inside the service call itself. // This allows moving work out of the single-threaded state service. @@ -137,25 +182,6 @@ impl From> for FinalizedBlock { } } -impl From for ContextuallyValidBlock { - fn from(prepared: PreparedBlock) -> Self { - let PreparedBlock { - block, - hash, - height, - new_outputs, - transaction_hashes, - } = prepared; - Self { - block, - hash, - height, - new_outputs: transparent::utxos_from_ordered_utxos(new_outputs), - transaction_hashes, - } - } -} - impl From for FinalizedBlock { fn from(contextually_valid: ContextuallyValidBlock) -> Self { let ContextuallyValidBlock { @@ -164,6 +190,7 @@ impl From for FinalizedBlock { height, new_outputs, transaction_hashes, + chain_value_pool_change: _, } = contextually_valid; Self { block, diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 97d6d9f5..5c954dee 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -420,8 +420,7 @@ impl FinalizedState { all_utxos_spent_by_block.extend(new_outputs); let current_pool = self.current_value_pool(); - let new_pool = - current_pool.update_with_block(block.borrow(), &all_utxos_spent_by_block)?; + let new_pool = current_pool.add_block(block.borrow(), &all_utxos_spent_by_block)?; batch.zs_insert(tip_chain_value_pool, (), new_pool); Ok(batch) @@ -614,6 +613,16 @@ impl FinalizedState { .zs_get(value_pool_cf, &()) .unwrap_or_else(ValueBalance::zero) } + + /// Allow to set up a fake value pool in the database for testing purposes. + #[cfg(any(test, feature = "proptest-impl"))] + #[allow(dead_code)] + pub fn set_current_value_pool(&self, fake_value_pool: ValueBalance) { + let mut batch = rocksdb::WriteBatch::default(); + let value_pool_cf = self.db.cf_handle("tip_chain_value_pool").unwrap(); + batch.zs_insert(value_pool_cf, (), fake_value_pool); + self.db.write(batch).unwrap(); + } } // Drop isn't guaranteed to run, such as when we panic, or if someone stored diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index f4209747..422558db 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -25,7 +25,10 @@ use zebra_chain::{ #[cfg(test)] use zebra_chain::sprout; -use crate::{FinalizedBlock, HashOrHeight, PreparedBlock, ValidateContextError}; +use crate::{ + request::ContextuallyValidBlock, FinalizedBlock, HashOrHeight, PreparedBlock, + ValidateContextError, +}; use self::chain::Chain; @@ -167,6 +170,7 @@ impl NonFinalizedState { finalized_state.sapling_note_commitment_tree(), finalized_state.orchard_note_commitment_tree(), finalized_state.history_tree(), + finalized_state.current_value_pool(), ); let (height, hash) = (prepared.height, prepared.hash); @@ -185,7 +189,7 @@ impl NonFinalizedState { prepared: PreparedBlock, finalized_state: &FinalizedState, ) -> Result { - check::utxo::transparent_spend( + let spent_utxos = check::utxo::transparent_spend( &prepared, &parent_chain.unspent_utxos(), &parent_chain.spent_utxos, @@ -197,7 +201,21 @@ impl NonFinalizedState { &parent_chain.history_tree, )?; - parent_chain.push(prepared) + let contextual = ContextuallyValidBlock::with_block_and_spent_utxos( + prepared.clone(), + spent_utxos.clone(), + ) + .map_err(|value_balance_error| { + ValidateContextError::CalculateBlockChainValueChange { + value_balance_error, + height: prepared.height, + block_hash: prepared.hash, + transaction_count: prepared.block.transactions.len(), + spent_utxo_count: spent_utxos.len(), + } + })?; + + parent_chain.push(contextual) } /// Returns the length of the non-finalized portion of the current best chain. @@ -352,7 +370,7 @@ impl NonFinalizedState { } /// Return the non-finalized portion of the current best chain - fn best_chain(&self) -> Option<&Chain> { + pub(crate) fn best_chain(&self) -> Option<&Chain> { self.chain_set .iter() .next_back() diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs index d0995cb0..2644bd84 100644 --- a/zebra-state/src/service/non_finalized_state/chain.rs +++ b/zebra-state/src/service/non_finalized_state/chain.rs @@ -8,12 +8,20 @@ use multiset::HashMultiSet; use tracing::instrument; use zebra_chain::{ - block, history_tree::HistoryTree, orchard, parameters::Network, primitives::Groth16Proof, - sapling, sprout, transaction, transaction::Transaction::*, transparent, + amount::{NegativeAllowed, NonNegative}, + block, + history_tree::HistoryTree, + orchard, + parameters::Network, + primitives::Groth16Proof, + sapling, sprout, transaction, + transaction::Transaction::*, + transparent, + value_balance::ValueBalance, work::difficulty::PartialCumulativeWork, }; -use crate::{service::check, ContextuallyValidBlock, PreparedBlock, ValidateContextError}; +use crate::{service::check, ContextuallyValidBlock, ValidateContextError}; #[derive(Debug, Clone)] pub struct Chain { @@ -35,20 +43,23 @@ pub struct Chain { /// including those created by earlier transactions or blocks in the chain. pub(crate) spent_utxos: HashSet, - /// The Sapling note commitment tree of the tip of this Chain. + /// The Sapling note commitment tree of the tip of this `Chain`, + /// including all finalized notes, and the non-finalized notes in this chain. pub(super) sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, - /// The Orchard note commitment tree of the tip of this Chain. + /// The Orchard note commitment tree of the tip of this `Chain`, + /// including all finalized notes, and the non-finalized notes in this chain. pub(super) orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, - /// The ZIP-221 history tree of the tip of this Chain. + /// The ZIP-221 history tree of the tip of this `Chain`, + /// including all finalized blocks, and the non-finalized `blocks` in this chain. pub(crate) history_tree: HistoryTree, /// The Sapling anchors created by `blocks`. pub(super) sapling_anchors: HashMultiSet, - /// The Sapling anchors created by each block in the chain. + /// The Sapling anchors created by each block in `blocks`. pub(super) sapling_anchors_by_height: BTreeMap, /// The Orchard anchors created by `blocks`. pub(super) orchard_anchors: HashMultiSet, - /// The Orchard anchors created by each block in the chain. + /// The Orchard anchors created by each block in `blocks`. pub(super) orchard_anchors_by_height: BTreeMap, /// The Sprout nullifiers revealed by `blocks`. @@ -58,8 +69,20 @@ pub struct Chain { /// The Orchard nullifiers revealed by `blocks`. pub(super) orchard_nullifiers: HashSet, - /// The cumulative work represented by this partial non-finalized chain. + /// The cumulative work represented by `blocks`. + /// + /// Since the best chain is determined by the largest cumulative work, + /// the work represented by finalized blocks can be ignored, + /// because they are common to all non-finalized chains. pub(super) partial_cumulative_work: PartialCumulativeWork, + + /// The chain value pool balances of the tip of this `Chain`, + /// including the block value pool changes from all finalized blocks, + /// and the non-finalized blocks in this chain. + /// + /// When a new chain is created from the finalized tip, + /// it is initialized with the finalized tip chain value pool balances. + pub(crate) chain_value_pools: ValueBalance, } impl Chain { @@ -69,6 +92,7 @@ impl Chain { sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, history_tree: HistoryTree, + finalized_tip_chain_value_pools: ValueBalance, ) -> Self { Self { network, @@ -88,6 +112,7 @@ impl Chain { orchard_nullifiers: Default::default(), partial_cumulative_work: Default::default(), history_tree, + chain_value_pools: finalized_tip_chain_value_pools, } } @@ -135,21 +160,25 @@ impl Chain { self.orchard_nullifiers == other.orchard_nullifiers && // proof of work - self.partial_cumulative_work == other.partial_cumulative_work + self.partial_cumulative_work == other.partial_cumulative_work && + + // chain value pool balances + self.chain_value_pools == other.chain_value_pools } /// Push a contextually valid non-finalized block into this chain as the new tip. /// /// If the block is invalid, drop this chain and return an error. + /// + /// Note: a [`ContextuallyValidBlock`] isn't actually contextually valid until + /// [`update_chain_state_with`] returns success. #[instrument(level = "debug", skip(self, block), fields(block = %block.block))] - pub fn push(mut self, block: PreparedBlock) -> Result { - // the block isn't contextually valid until `update_chain_state_with` returns success - let block = ContextuallyValidBlock::from(block); - + pub fn push(mut self, block: ContextuallyValidBlock) -> Result { // update cumulative data members - self.update_chain_state_with(&block)?; + self.update_chain_tip_with(&block)?; tracing::debug!(block = %block.block, "adding block to chain"); self.blocks.insert(block.height, block); + Ok(self) } @@ -165,7 +194,7 @@ impl Chain { .expect("only called while blocks is populated"); // update cumulative data members - self.revert_chain_state_with(&block); + self.revert_chain_with(&block, RevertPosition::Root); // return the prepared block block @@ -269,17 +298,26 @@ impl Chain { "Non-finalized chains must have at least one block to be valid" ); - self.revert_chain_state_with(&block); + self.revert_chain_with(&block, RevertPosition::Tip); } + /// Return the non-finalized tip height for this chain. + /// + /// # Panics + /// + /// Panics if called while the chain is empty, + /// or while the chain is updating its internal state with the first block. pub fn non_finalized_tip_height(&self) -> block::Height { - *self - .blocks - .keys() - .next_back() + self.max_block_height() .expect("only called while blocks is populated") } + /// Return the non-finalized tip height for this chain, + /// or `None` if `self.blocks` is empty. + fn max_block_height(&self) -> Option { + self.blocks.keys().next_back().cloned() + } + pub fn is_empty(&self) -> bool { self.blocks.is_empty() } @@ -323,39 +361,54 @@ impl Chain { orchard_nullifiers: self.orchard_nullifiers.clone(), partial_cumulative_work: self.partial_cumulative_work, history_tree, + chain_value_pools: self.chain_value_pools, } } } -/// Helper trait to organize inverse operations done on the `Chain` type. Used to -/// overload the `update_chain_state_with` and `revert_chain_state_with` methods -/// based on the type of the argument. -/// -/// This trait was motivated by the length of the `push` and `pop_root` functions -/// and fear that it would be easy to introduce bugs when updating them unless -/// the code was reorganized to keep related operations adjacent to eachother. -trait UpdateWith { - /// Update `Chain` cumulative data members to add data that are derived from - /// `T` - fn update_chain_state_with(&mut self, _: &T) -> Result<(), ValidateContextError>; +/// The revert position being performed on a chain. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +enum RevertPosition { + /// The chain root is being reverted via [`pop_root`], + /// when a block is finalized. + Root, - /// Update `Chain` cumulative data members to remove data that are derived - /// from `T` - fn revert_chain_state_with(&mut self, _: &T); + /// The chain tip is being reverted via [`pop_tip`], + /// when a chain is forked. + Tip, +} + +/// Helper trait to organize inverse operations done on the `Chain` type. +/// +/// Used to overload update and revert methods, based on the type of the argument, +/// and the position of the removed block in the chain. +/// +/// This trait was motivated by the length of the `push`, `pop_root`, and `pop_tip` functions, +/// and fear that it would be easy to introduce bugs when updating them, +/// unless the code was reorganized to keep related operations adjacent to each other. +trait UpdateWith { + /// When `T` is added to the chain tip, + /// update `Chain` cumulative data members to add data that are derived from `T`. + fn update_chain_tip_with(&mut self, _: &T) -> Result<(), ValidateContextError>; + + /// When `T` is removed from `position` in the chain, + /// revert `Chain` cumulative data members to remove data that are derived from `T`. + fn revert_chain_with(&mut self, _: &T, position: RevertPosition); } impl UpdateWith for Chain { #[instrument(skip(self, contextually_valid), fields(block = %contextually_valid.block))] - fn update_chain_state_with( + fn update_chain_tip_with( &mut self, contextually_valid: &ContextuallyValidBlock, ) -> Result<(), ValidateContextError> { - let (block, hash, height, new_outputs, transaction_hashes) = ( + let (block, hash, height, new_outputs, transaction_hashes, chain_value_pool_change) = ( contextually_valid.block.as_ref(), contextually_valid.hash, contextually_valid.height, &contextually_valid.new_outputs, &contextually_valid.transaction_hashes, + &contextually_valid.chain_value_pool_change, ); // add hash to height_by_hash @@ -420,15 +473,15 @@ impl UpdateWith for Chain { ); // add the utxos this produced - self.update_chain_state_with(new_outputs)?; + self.update_chain_tip_with(new_outputs)?; // add the utxos this consumed - self.update_chain_state_with(inputs)?; + self.update_chain_tip_with(inputs)?; // add the shielded data - self.update_chain_state_with(joinsplit_data)?; - self.update_chain_state_with(sapling_shielded_data_per_spend_anchor)?; - self.update_chain_state_with(sapling_shielded_data_shared_anchor)?; - self.update_chain_state_with(orchard_shielded_data)?; + self.update_chain_tip_with(joinsplit_data)?; + self.update_chain_tip_with(sapling_shielded_data_per_spend_anchor)?; + self.update_chain_tip_with(sapling_shielded_data_shared_anchor)?; + self.update_chain_tip_with(orchard_shielded_data)?; } // Having updated all the note commitment trees and nullifier sets in @@ -448,17 +501,25 @@ impl UpdateWith for Chain { orchard_root, )?; + // update the chain value pool balances + self.update_chain_tip_with(chain_value_pool_change)?; + Ok(()) } #[instrument(skip(self, contextually_valid), fields(block = %contextually_valid.block))] - fn revert_chain_state_with(&mut self, contextually_valid: &ContextuallyValidBlock) { - let (block, hash, height, new_outputs, transaction_hashes) = ( + fn revert_chain_with( + &mut self, + contextually_valid: &ContextuallyValidBlock, + position: RevertPosition, + ) { + let (block, hash, height, new_outputs, transaction_hashes, chain_value_pool_change) = ( contextually_valid.block.as_ref(), contextually_valid.hash, contextually_valid.height, &contextually_valid.new_outputs, &contextually_valid.transaction_hashes, + &contextually_valid.chain_value_pool_change, ); // remove the blocks hash from `height_by_hash` @@ -521,16 +582,17 @@ impl UpdateWith for Chain { ); // remove the utxos this produced - self.revert_chain_state_with(new_outputs); + self.revert_chain_with(new_outputs, position); // remove the utxos this consumed - self.revert_chain_state_with(inputs); + self.revert_chain_with(inputs, position); // remove the shielded data - self.revert_chain_state_with(joinsplit_data); - self.revert_chain_state_with(sapling_shielded_data_per_spend_anchor); - self.revert_chain_state_with(sapling_shielded_data_shared_anchor); - self.revert_chain_state_with(orchard_shielded_data); + self.revert_chain_with(joinsplit_data, position); + self.revert_chain_with(sapling_shielded_data_per_spend_anchor, position); + self.revert_chain_with(sapling_shielded_data_shared_anchor, position); + self.revert_chain_with(orchard_shielded_data, position); } + let anchor = self .sapling_anchors_by_height .remove(&height) @@ -547,11 +609,14 @@ impl UpdateWith for Chain { self.orchard_anchors.remove(&anchor), "Orchard anchor must be present if block was added to chain" ); + + // revert the chain value pool balances, if needed + self.revert_chain_with(chain_value_pool_change, position); } } impl UpdateWith> for Chain { - fn update_chain_state_with( + fn update_chain_tip_with( &mut self, utxos: &HashMap, ) -> Result<(), ValidateContextError> { @@ -560,9 +625,10 @@ impl UpdateWith> for Chain { Ok(()) } - fn revert_chain_state_with( + fn revert_chain_with( &mut self, utxos: &HashMap, + _position: RevertPosition, ) { self.created_utxos .retain(|outpoint, _| !utxos.contains_key(outpoint)); @@ -570,7 +636,7 @@ impl UpdateWith> for Chain { } impl UpdateWith> for Chain { - fn update_chain_state_with( + fn update_chain_tip_with( &mut self, inputs: &Vec, ) -> Result<(), ValidateContextError> { @@ -585,7 +651,7 @@ impl UpdateWith> for Chain { Ok(()) } - fn revert_chain_state_with(&mut self, inputs: &Vec) { + fn revert_chain_with(&mut self, inputs: &Vec, _position: RevertPosition) { for consumed_utxo in inputs { match consumed_utxo { transparent::Input::PrevOut { outpoint, .. } => { @@ -602,7 +668,7 @@ impl UpdateWith> for Chain { impl UpdateWith>> for Chain { #[instrument(skip(self, joinsplit_data))] - fn update_chain_state_with( + fn update_chain_tip_with( &mut self, joinsplit_data: &Option>, ) -> Result<(), ValidateContextError> { @@ -621,9 +687,10 @@ impl UpdateWith>> for Chain { /// /// See [`check::nullifier::remove_from_non_finalized_chain`] for details. #[instrument(skip(self, joinsplit_data))] - fn revert_chain_state_with( + fn revert_chain_with( &mut self, joinsplit_data: &Option>, + _position: RevertPosition, ) { if let Some(joinsplit_data) = joinsplit_data { check::nullifier::remove_from_non_finalized_chain( @@ -639,7 +706,7 @@ where AnchorV: sapling::AnchorVariant + Clone, { #[instrument(skip(self, sapling_shielded_data))] - fn update_chain_state_with( + fn update_chain_tip_with( &mut self, sapling_shielded_data: &Option>, ) -> Result<(), ValidateContextError> { @@ -662,9 +729,10 @@ where /// /// See [`check::nullifier::remove_from_non_finalized_chain`] for details. #[instrument(skip(self, sapling_shielded_data))] - fn revert_chain_state_with( + fn revert_chain_with( &mut self, sapling_shielded_data: &Option>, + _position: RevertPosition, ) { if let Some(sapling_shielded_data) = sapling_shielded_data { // Note commitments are not removed from the tree here because we @@ -681,7 +749,7 @@ where impl UpdateWith> for Chain { #[instrument(skip(self, orchard_shielded_data))] - fn update_chain_state_with( + fn update_chain_tip_with( &mut self, orchard_shielded_data: &Option, ) -> Result<(), ValidateContextError> { @@ -704,7 +772,11 @@ impl UpdateWith> for Chain { /// /// See [`check::nullifier::remove_from_non_finalized_chain`] for details. #[instrument(skip(self, orchard_shielded_data))] - fn revert_chain_state_with(&mut self, orchard_shielded_data: &Option) { + fn revert_chain_with( + &mut self, + orchard_shielded_data: &Option, + _position: RevertPosition, + ) { if let Some(orchard_shielded_data) = orchard_shielded_data { // Note commitments are not removed from the tree here because we // don't support that operation yet. Instead, we recreate the tree @@ -718,6 +790,57 @@ impl UpdateWith> for Chain { } } +impl UpdateWith> for Chain { + fn update_chain_tip_with( + &mut self, + block_value_pool_change: &ValueBalance, + ) -> Result<(), ValidateContextError> { + match self + .chain_value_pools + .add_chain_value_pool_change(*block_value_pool_change) + { + Ok(chain_value_pools) => self.chain_value_pools = chain_value_pools, + Err(value_balance_error) => Err(ValidateContextError::AddValuePool { + value_balance_error, + chain_value_pools: self.chain_value_pools, + block_value_pool_change: *block_value_pool_change, + // assume that the current block is added to `blocks` after `update_chain_tip_with` + height: self.max_block_height().and_then(|height| height + 1), + })?, + }; + + Ok(()) + } + + /// Revert the chain state using a block chain value pool change. + /// + /// When forking from the tip, subtract the block's chain value pool change. + /// + /// When finalizing the root, leave the chain value pool balances unchanged. + /// [`chain_value_pools`] tracks the chain value pools for all finalized blocks, + /// and the non-finalized blocks in this chain. + /// So finalizing the root doesn't change the set of blocks it tracks. + /// + /// # Panics + /// + /// Panics if the chain pool value balance is invalid + /// after we subtract the block value pool change. + fn revert_chain_with( + &mut self, + block_value_pool_change: &ValueBalance, + position: RevertPosition, + ) { + use std::ops::Neg; + + if position == RevertPosition::Tip { + self.chain_value_pools = self + .chain_value_pools + .add_chain_value_pool_change(block_value_pool_change.neg()) + .expect("reverting the tip will leave the pools in a previously valid state"); + } + } +} + impl Ord for Chain { /// Chain order for the [`NonFinalizedState`]'s `chain_set`. /// Chains with higher cumulative Proof of Work are [`Ordering::Greater`], diff --git a/zebra-state/src/service/non_finalized_state/tests/prop.rs b/zebra-state/src/service/non_finalized_state/tests/prop.rs index cc066e9e..7bf994d1 100644 --- a/zebra-state/src/service/non_finalized_state/tests/prop.rs +++ b/zebra-state/src/service/non_finalized_state/tests/prop.rs @@ -1,18 +1,21 @@ -use std::{env, sync::Arc}; +use std::{collections::BTreeMap, env, sync::Arc}; use zebra_test::prelude::*; use zebra_chain::{ + amount::NonNegative, block::{self, arbitrary::allow_all_transparent_coinbase_spends, Block}, fmt::DisplayToDebug, history_tree::{HistoryTree, NonEmptyHistoryTree}, parameters::NetworkUpgrade::*, parameters::{Network, *}, + value_balance::ValueBalance, LedgerState, }; use crate::{ arbitrary::Prepare, + request::ContextuallyValidBlock, service::{ arbitrary::PreparedChain, finalized_state::FinalizedState, @@ -21,35 +24,136 @@ use crate::{ Config, }; +/// The default number of proptest cases for long partial chain tests. const DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES: u32 = 1; -/// Check that a forked chain is the same as a chain that had the same blocks appended. +/// The default number of proptest cases for short partial chain tests. +const DEFAULT_SHORT_CHAIN_PROPTEST_CASES: u32 = 16; + +/// Check that chain block pushes work with blocks from genesis /// -/// Also check for: -/// - no transparent spends in the genesis block, because genesis transparent outputs are ignored +/// Logs extra debugging information when the chain value balances fail. #[test] -fn forked_equals_pushed() -> Result<()> { +fn push_genesis_chain() -> Result<()> { zebra_test::init(); - proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), - |((chain, fork_at_count, network, finalized_tree) in PreparedChain::new_heartwood())| { - // Skip first block which was used for the history tree; make sure fork_at_count is still valid - let fork_at_count = std::cmp::min(fork_at_count, chain.len() - 1); - let chain = &chain[1..]; - // use `fork_at_count` as the fork tip - let fork_tip_hash = chain[fork_at_count - 1].hash; + proptest!( + ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, count, network, empty_tree) in PreparedChain::default())| { + prop_assert!(empty_tree.is_none()); - let mut full_chain = Chain::new(network, Default::default(), Default::default(), finalized_tree.clone()); - let mut partial_chain = Chain::new(network, Default::default(), Default::default(), finalized_tree.clone()); + let mut only_chain = Chain::new(network, Default::default(), Default::default(), empty_tree, ValueBalance::zero()); + // contains the block value pool changes and chain value pool balances for each height + let mut chain_values = BTreeMap::new(); - for block in chain.iter().take(fork_at_count) { - partial_chain = partial_chain.push(block.clone())?; + chain_values.insert(None, (None, only_chain.chain_value_pools.into())); + + for block in chain.iter().take(count).cloned() { + let block = + ContextuallyValidBlock::with_block_and_spent_utxos( + block, + only_chain.unspent_utxos(), + ) + .map_err(|e| (e, chain_values.clone())) + .expect("invalid block value pool change"); + + chain_values.insert(block.height.into(), (block.chain_value_pool_change.into(), None)); + + only_chain = only_chain + .push(block.clone()) + .map_err(|e| (e, chain_values.clone())) + .expect("invalid chain value pools"); + + chain_values.insert(block.height.into(), (block.chain_value_pool_change.into(), only_chain.chain_value_pools.into())); } - for block in chain.iter() { - full_chain = full_chain.push(block.clone())?; + + prop_assert_eq!(only_chain.blocks.len(), count); + }); + + Ok(()) +} + +/// Check that chain block pushes work with history tree blocks +#[test] +fn push_history_tree_chain() -> Result<()> { + zebra_test::init(); + + proptest!( + ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, count, network, finalized_tree) in PreparedChain::new_heartwood())| { + prop_assert!(finalized_tree.is_some()); + + // Skip first block which was used for the history tree. + // This skips some transactions which are required to calculate value balances, + // so we zero all transparent inputs in this test. + + // make sure count is still valid + let count = std::cmp::min(count, chain.len() - 1); + let chain = &chain[1..]; + + let mut only_chain = Chain::new(network, Default::default(), Default::default(), finalized_tree, ValueBalance::zero()); + + for block in chain + .iter() + .take(count) + .map(ContextuallyValidBlock::test_with_zero_chain_pool_change) { + only_chain = only_chain.push(block)?; + } + + prop_assert_eq!(only_chain.blocks.len(), count); + }); + + Ok(()) +} + +/// Check that a forked genesis chain is the same as a chain that had the same blocks appended. +/// +/// Also check that: +/// - there are no transparent spends in the chain from the genesis block, +/// because genesis transparent outputs are ignored +/// - transactions only spend transparent outputs from earlier in the block or chain +/// - chain value balances are non-negative +#[test] +fn forked_equals_pushed_genesis() -> Result<()> { + zebra_test::init(); + + proptest!( + ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, fork_at_count, network, empty_tree) in PreparedChain::default())| { + + prop_assert!(empty_tree.is_none()); + + // use `fork_at_count` as the fork tip + let fork_tip_hash = chain[fork_at_count - 1].hash; + + let mut full_chain = Chain::new(network, Default::default(), Default::default(), empty_tree.clone(), ValueBalance::zero()); + let mut partial_chain = Chain::new(network, Default::default(), Default::default(), empty_tree.clone(), ValueBalance::zero()); + + for block in chain.iter().take(fork_at_count).cloned() { + let block = + ContextuallyValidBlock::with_block_and_spent_utxos( + block, + partial_chain.unspent_utxos(), + )?; + partial_chain = partial_chain.push(block).expect("partial chain push is valid"); + } + + for block in chain.iter().cloned() { + let block = + ContextuallyValidBlock::with_block_and_spent_utxos( + block, + full_chain.unspent_utxos(), + )?; + full_chain = full_chain.push(block.clone()).expect("full chain push is valid"); // check some other properties of generated chains if block.height == block::Height(0) { @@ -70,66 +174,157 @@ fn forked_equals_pushed() -> Result<()> { } } - let mut forked = full_chain - .fork( - fork_tip_hash, - Default::default(), - Default::default(), - finalized_tree, - ) - .expect("fork works") - .expect("hash is present"); + let mut forked = full_chain + .fork( + fork_tip_hash, + Default::default(), + Default::default(), + empty_tree, + ) + .expect("fork works") + .expect("hash is present"); - // the first check is redundant, but it's useful for debugging - prop_assert_eq!(forked.blocks.len(), partial_chain.blocks.len()); - prop_assert!(forked.eq_internal_state(&partial_chain)); + // the first check is redundant, but it's useful for debugging + prop_assert_eq!(forked.blocks.len(), partial_chain.blocks.len()); + prop_assert!(forked.eq_internal_state(&partial_chain)); - // Re-add blocks to the fork and check if we arrive at the - // same original full chain - for block in chain.iter().skip(fork_at_count) { - forked = forked.push(block.clone())?; - } + // Re-add blocks to the fork and check if we arrive at the + // same original full chain + for block in chain.iter().skip(fork_at_count).cloned() { + let block = + ContextuallyValidBlock::with_block_and_spent_utxos( + block, + forked.unspent_utxos(), + )?; + forked = forked.push(block).expect("forked chain push is valid"); + } - prop_assert_eq!(forked.blocks.len(), full_chain.blocks.len()); - prop_assert!(forked.eq_internal_state(&full_chain)); - }); + prop_assert_eq!(forked.blocks.len(), full_chain.blocks.len()); + prop_assert!(forked.eq_internal_state(&full_chain)); + }); Ok(()) } -/// Check that a chain with some blocks finalized is the same as +/// Check that a forked history tree chain is the same as a chain that had the same blocks appended. +#[test] +fn forked_equals_pushed_history_tree() -> Result<()> { + zebra_test::init(); + + proptest!( + ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, fork_at_count, network, finalized_tree) in PreparedChain::new_heartwood())| { + prop_assert!(finalized_tree.is_some()); + + // Skip first block which was used for the history tree. + // This skips some transactions which are required to calculate value balances, + // so we zero all transparent inputs in this test. + + // make sure fork_at_count is still valid + let fork_at_count = std::cmp::min(fork_at_count, chain.len() - 1); + let chain = &chain[1..]; + // use `fork_at_count` as the fork tip + let fork_tip_hash = chain[fork_at_count - 1].hash; + + let mut full_chain = Chain::new(network, Default::default(), Default::default(), finalized_tree.clone(), ValueBalance::zero()); + let mut partial_chain = Chain::new(network, Default::default(), Default::default(), finalized_tree.clone(), ValueBalance::zero()); + + for block in chain + .iter() + .take(fork_at_count) + .map(ContextuallyValidBlock::test_with_zero_chain_pool_change) { + partial_chain = partial_chain.push(block)?; + } + + for block in chain + .iter() + .map(ContextuallyValidBlock::test_with_zero_chain_pool_change) { + full_chain = full_chain.push(block.clone())?; + } + + let mut forked = full_chain + .fork( + fork_tip_hash, + Default::default(), + Default::default(), + finalized_tree, + ) + .expect("fork works") + .expect("hash is present"); + + // the first check is redundant, but it's useful for debugging + prop_assert_eq!(forked.blocks.len(), partial_chain.blocks.len()); + prop_assert!(forked.eq_internal_state(&partial_chain)); + + // Re-add blocks to the fork and check if we arrive at the + // same original full chain + for block in chain + .iter() + .skip(fork_at_count) + .map(ContextuallyValidBlock::test_with_zero_chain_pool_change) { + forked = forked.push(block)?; + } + + prop_assert_eq!(forked.blocks.len(), full_chain.blocks.len()); + prop_assert!(forked.eq_internal_state(&full_chain)); + }); + + Ok(()) +} + +/// Check that a genesis chain with some blocks finalized is the same as /// a chain that never had those blocks added. #[test] -fn finalized_equals_pushed() -> Result<()> { +fn finalized_equals_pushed_genesis() -> Result<()> { zebra_test::init(); proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), - |((chain, end_count, network, finalized_tree) in PreparedChain::new_heartwood())| { - // Skip first block which was used for the history tree; make sure end_count is still valid - let end_count = std::cmp::min(end_count, chain.len() - 1); - let chain = &chain[1..]; + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, end_count, network, empty_tree) in PreparedChain::default())| { + + // This test starts a partial chain from the middle of `chain`, + // so it doesn't have the unspent UTXOs needed to calculate value balances. + + prop_assert!(empty_tree.is_none()); + // use `end_count` as the number of non-finalized blocks at the end of the chain let finalized_count = chain.len() - end_count; - let mut full_chain = Chain::new(network, Default::default(), Default::default(), finalized_tree); - for block in chain.iter().take(finalized_count) { - full_chain = full_chain.push(block.clone())?; - } + let fake_value_pool = ValueBalance::::fake_populated_pool(); + + let mut full_chain = Chain::new(network, Default::default(), Default::default(), empty_tree, fake_value_pool); + for block in chain + .iter() + .take(finalized_count) + .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { + full_chain = full_chain.push(block)?; + } + let mut partial_chain = Chain::new( network, full_chain.sapling_note_commitment_tree.clone(), full_chain.orchard_note_commitment_tree.clone(), full_chain.history_tree.clone(), + full_chain.chain_value_pools, ); - for block in chain.iter().skip(finalized_count) { - partial_chain = partial_chain.push(block.clone())?; - } - for block in chain.iter().skip(finalized_count) { - full_chain = full_chain.push(block.clone())?; - } + for block in chain + .iter() + .skip(finalized_count) + .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { + partial_chain = partial_chain.push(block.clone())?; + } + + for block in chain + .iter() + .skip(finalized_count) + .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { + full_chain = full_chain.push(block.clone())?; + } for _ in 0..finalized_count { let _finalized = full_chain.pop_root(); @@ -142,86 +337,158 @@ fn finalized_equals_pushed() -> Result<()> { Ok(()) } -/// Check that rejected blocks do not change the internal state of a chain -/// in a non-finalized state. +/// Check that a history tree chain with some blocks finalized is the same as +/// a chain that never had those blocks added. #[test] -fn rejection_restores_internal_state() -> Result<()> { +fn finalized_equals_pushed_history_tree() -> Result<()> { zebra_test::init(); proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), - |((chain, valid_count, network, mut bad_block) in (PreparedChain::default(), any::(), any::()) - .prop_flat_map(|((chain, valid_count, network, _history_tree), is_nu5, is_v5)| { - let next_height = chain[valid_count - 1].height; - ( - Just(chain), - Just(valid_count), - Just(network), - // generate a Canopy or NU5 block with v4 or v5 transactions - LedgerState::height_strategy( - next_height, - if is_nu5 { Nu5 } else { Canopy }, - if is_nu5 && is_v5 { 5 } else { 4 }, - true, - ) - .prop_flat_map(Block::arbitrary_with) - .prop_map(DisplayToDebug) - ) - } - ))| { - let mut state = NonFinalizedState::new(network); - let finalized_state = FinalizedState::new(&Config::ephemeral(), network); + |((chain, end_count, network, finalized_tree) in PreparedChain::new_heartwood())| { - // use `valid_count` as the number of valid blocks before an invalid block - let valid_tip_height = chain[valid_count - 1].height; - let valid_tip_hash = chain[valid_count - 1].hash; - let mut chain = chain.iter().take(valid_count).cloned(); - prop_assert!(state.eq_internal_state(&state)); + prop_assert!(finalized_tree.is_some()); - if let Some(first_block) = chain.next() { - let result = state.commit_new_chain(first_block, &finalized_state); - prop_assert_eq!( - result, - Ok(()), - "PreparedChain should generate a valid first block" - ); - prop_assert!(state.eq_internal_state(&state)); - } + // Skip first block which was used for the history tree; make sure end_count is still valid + // + // This skips some transactions which are required to calculate value balances, + // so we zero all transparent inputs in this test. + // + // This test also starts a partial chain from the middle of `chain`, + // so it doesn't have the unspent UTXOs needed to calculate value balances. + let end_count = std::cmp::min(end_count, chain.len() - 1); + let chain = &chain[1..]; + // use `end_count` as the number of non-finalized blocks at the end of the chain + let finalized_count = chain.len() - end_count; - for block in chain { - let result = state.commit_block(block.clone(), &finalized_state); - prop_assert_eq!( - result, - Ok(()), - "PreparedChain should generate a valid block at {:?}", - block.height, - ); - prop_assert!(state.eq_internal_state(&state)); - } + let fake_value_pool = ValueBalance::::fake_populated_pool(); - prop_assert_eq!(state.best_tip(), Some((valid_tip_height, valid_tip_hash))); + let mut full_chain = Chain::new(network, Default::default(), Default::default(), finalized_tree, fake_value_pool); + for block in chain + .iter() + .take(finalized_count) + .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { + full_chain = full_chain.push(block)?; + } - let mut reject_state = state.clone(); - // the tip check is redundant, but it's useful for debugging - prop_assert_eq!(state.best_tip(), reject_state.best_tip()); - prop_assert!(state.eq_internal_state(&reject_state)); + let mut partial_chain = Chain::new( + network, + full_chain.sapling_note_commitment_tree.clone(), + full_chain.orchard_note_commitment_tree.clone(), + full_chain.history_tree.clone(), + full_chain.chain_value_pools, + ); + for block in chain + .iter() + .skip(finalized_count) + .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { + partial_chain = partial_chain.push(block.clone())?; + } - bad_block.header.previous_block_hash = valid_tip_hash; - let bad_block = Arc::new(bad_block.0).prepare(); - let reject_result = reject_state.commit_block(bad_block, &finalized_state); + for block in chain + .iter() + .skip(finalized_count) + .map(ContextuallyValidBlock::test_with_zero_spent_utxos) { + full_chain = full_chain.push(block.clone())?; + } - if reject_result.is_err() { - prop_assert_eq!(state.best_tip(), reject_state.best_tip()); - prop_assert!(state.eq_internal_state(&reject_state)); - } else { - // the block just happened to pass all the non-finalized checks - prop_assert_ne!(state.best_tip(), reject_state.best_tip()); - prop_assert!(!state.eq_internal_state(&reject_state)); - } - }); + for _ in 0..finalized_count { + let _finalized = full_chain.pop_root(); + } + + prop_assert_eq!(full_chain.blocks.len(), partial_chain.blocks.len()); + prop_assert!(full_chain.eq_internal_state(&partial_chain)); + }); + + Ok(()) +} + +/// Check that rejected blocks do not change the internal state of a genesis chain +/// in a non-finalized state. +#[test] +fn rejection_restores_internal_state_genesis() -> Result<()> { + zebra_test::init(); + + proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + |((chain, valid_count, network, mut bad_block) in (PreparedChain::default(), any::(), any::()) + .prop_flat_map(|((chain, valid_count, network, _history_tree), is_nu5, is_v5)| { + let next_height = chain[valid_count - 1].height; + ( + Just(chain), + Just(valid_count), + Just(network), + // generate a Canopy or NU5 block with v4 or v5 transactions + LedgerState::height_strategy( + next_height, + if is_nu5 { Nu5 } else { Canopy }, + if is_nu5 && is_v5 { 5 } else { 4 }, + true, + ) + .prop_flat_map(Block::arbitrary_with) + .prop_map(DisplayToDebug) + ) + } + ))| { + let mut state = NonFinalizedState::new(network); + let finalized_state = FinalizedState::new(&Config::ephemeral(), network); + + let fake_value_pool = ValueBalance::::fake_populated_pool(); + finalized_state.set_current_value_pool(fake_value_pool); + + // use `valid_count` as the number of valid blocks before an invalid block + let valid_tip_height = chain[valid_count - 1].height; + let valid_tip_hash = chain[valid_count - 1].hash; + let mut chain = chain.iter().take(valid_count).cloned(); + + prop_assert!(state.eq_internal_state(&state)); + + if let Some(first_block) = chain.next() { + let result = state.commit_new_chain(first_block, &finalized_state); + prop_assert_eq!( + result, + Ok(()), + "PreparedChain should generate a valid first block" + ); + prop_assert!(state.eq_internal_state(&state)); + } + + for block in chain { + let result = state.commit_block(block.clone(), &finalized_state); + prop_assert_eq!( + result, + Ok(()), + "PreparedChain should generate a valid block at {:?}", + block.height, + ); + prop_assert!(state.eq_internal_state(&state)); + } + + prop_assert_eq!(state.best_tip(), Some((valid_tip_height, valid_tip_hash))); + + let mut reject_state = state.clone(); + // the tip check is redundant, but it's useful for debugging + prop_assert_eq!(state.best_tip(), reject_state.best_tip()); + prop_assert!(state.eq_internal_state(&reject_state)); + + bad_block.header.previous_block_hash = valid_tip_hash; + let bad_block = Arc::new(bad_block.0).prepare(); + let reject_result = reject_state.commit_block(bad_block, &finalized_state); + + if reject_result.is_err() { + prop_assert_eq!(state.best_tip(), reject_state.best_tip()); + prop_assert!(state.eq_internal_state(&reject_state)); + } else { + // the block just happened to pass all the non-finalized checks + prop_assert_ne!(state.best_tip(), reject_state.best_tip()); + prop_assert!(!state.eq_internal_state(&reject_state)); + } + }); Ok(()) } @@ -235,7 +502,7 @@ fn different_blocks_different_chains() -> Result<()> { proptest!(ProptestConfig::with_cases(env::var("PROPTEST_CASES") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(DEFAULT_PARTIAL_CHAIN_PROPTEST_CASES)), + .unwrap_or(DEFAULT_SHORT_CHAIN_PROPTEST_CASES)), |((vec1, vec2) in (any::(), any::()) .prop_flat_map(|(is_nu5, is_v5)| { // generate a Canopy or NU5 block with v4 or v5 transactions @@ -261,11 +528,11 @@ fn different_blocks_different_chains() -> Result<()> { } else { Default::default() }; - let chain1 = Chain::new(Network::Mainnet, Default::default(), Default::default(), finalized_tree1); - let chain2 = Chain::new(Network::Mainnet, Default::default(), Default::default(), finalized_tree2); + let chain1 = Chain::new(Network::Mainnet, Default::default(), Default::default(), finalized_tree1, ValueBalance::fake_populated_pool()); + let chain2 = Chain::new(Network::Mainnet, Default::default(), Default::default(), finalized_tree2, ValueBalance::fake_populated_pool()); - let block1 = vec1[1].clone().prepare(); - let block2 = vec2[1].clone().prepare(); + let block1 = vec1[1].clone().prepare().test_with_zero_spent_utxos(); + let block2 = vec2[1].clone().prepare().test_with_zero_spent_utxos(); let result1 = chain1.push(block1.clone()); let result2 = chain2.push(block2.clone()); @@ -317,6 +584,9 @@ fn different_blocks_different_chains() -> Result<()> { // proof of work chain1.partial_cumulative_work = chain2.partial_cumulative_work; + // chain value pool + chain1.chain_value_pools = chain2.chain_value_pools; + // If this check fails, the `Chain` fields are out // of sync with `eq_internal_state` or this test. prop_assert!( diff --git a/zebra-state/src/service/non_finalized_state/tests/vectors.rs b/zebra-state/src/service/non_finalized_state/tests/vectors.rs index 81eb41f9..85d2dc4c 100644 --- a/zebra-state/src/service/non_finalized_state/tests/vectors.rs +++ b/zebra-state/src/service/non_finalized_state/tests/vectors.rs @@ -1,10 +1,12 @@ use std::sync::Arc; use zebra_chain::{ + amount::NonNegative, block::Block, history_tree::NonEmptyHistoryTree, parameters::{Network, NetworkUpgrade}, serialization::ZcashDeserializeInto, + value_balance::ValueBalance, }; use zebra_test::prelude::*; @@ -28,6 +30,7 @@ fn construct_empty() { Default::default(), Default::default(), Default::default(), + ValueBalance::zero(), ); } @@ -42,8 +45,9 @@ fn construct_single() -> Result<()> { Default::default(), Default::default(), Default::default(), + ValueBalance::fake_populated_pool(), ); - chain = chain.push(block.prepare())?; + chain = chain.push(block.prepare().test_with_zero_spent_utxos())?; assert_eq!(1, chain.blocks.len()); @@ -69,10 +73,11 @@ fn construct_many() -> Result<()> { Default::default(), Default::default(), Default::default(), + ValueBalance::fake_populated_pool(), ); for block in blocks { - chain = chain.push(block.prepare())?; + chain = chain.push(block.prepare().test_with_zero_spent_utxos())?; } assert_eq!(100, chain.blocks.len()); @@ -93,16 +98,18 @@ fn ord_matches_work() -> Result<()> { Default::default(), Default::default(), Default::default(), + ValueBalance::fake_populated_pool(), ); - lesser_chain = lesser_chain.push(less_block.prepare())?; + lesser_chain = lesser_chain.push(less_block.prepare().test_with_zero_spent_utxos())?; let mut bigger_chain = Chain::new( Network::Mainnet, Default::default(), Default::default(), Default::default(), + ValueBalance::zero(), ); - bigger_chain = bigger_chain.push(more_block.prepare())?; + bigger_chain = bigger_chain.push(more_block.prepare().test_with_zero_spent_utxos())?; assert!(bigger_chain > lesser_chain); @@ -178,6 +185,9 @@ fn finalize_pops_from_best_chain_for_network(network: Network) -> Result<()> { let mut state = NonFinalizedState::new(network); let finalized_state = FinalizedState::new(&Config::ephemeral(), network); + let fake_value_pool = ValueBalance::::fake_populated_pool(); + finalized_state.set_current_value_pool(fake_value_pool); + state.commit_new_chain(block1.clone().prepare(), &finalized_state)?; state.commit_block(block2.clone().prepare(), &finalized_state)?; state.commit_block(child.prepare(), &finalized_state)?; @@ -226,6 +236,9 @@ fn commit_block_extending_best_chain_doesnt_drop_worst_chains_for_network( let mut state = NonFinalizedState::new(network); let finalized_state = FinalizedState::new(&Config::ephemeral(), network); + let fake_value_pool = ValueBalance::::fake_populated_pool(); + finalized_state.set_current_value_pool(fake_value_pool); + assert_eq!(0, state.chain_set.len()); state.commit_new_chain(block1.prepare(), &finalized_state)?; assert_eq!(1, state.chain_set.len()); @@ -270,6 +283,9 @@ fn shorter_chain_can_be_best_chain_for_network(network: Network) -> Result<()> { let mut state = NonFinalizedState::new(network); let finalized_state = FinalizedState::new(&Config::ephemeral(), network); + let fake_value_pool = ValueBalance::::fake_populated_pool(); + finalized_state.set_current_value_pool(fake_value_pool); + state.commit_new_chain(block1.prepare(), &finalized_state)?; state.commit_block(long_chain_block1.prepare(), &finalized_state)?; state.commit_block(long_chain_block2.prepare(), &finalized_state)?; @@ -314,6 +330,9 @@ fn longer_chain_with_more_work_wins_for_network(network: Network) -> Result<()> let mut state = NonFinalizedState::new(network); let finalized_state = FinalizedState::new(&Config::ephemeral(), network); + let fake_value_pool = ValueBalance::::fake_populated_pool(); + finalized_state.set_current_value_pool(fake_value_pool); + state.commit_new_chain(block1.prepare(), &finalized_state)?; state.commit_block(long_chain_block1.prepare(), &finalized_state)?; state.commit_block(long_chain_block2.prepare(), &finalized_state)?; @@ -356,6 +375,9 @@ fn equal_length_goes_to_more_work_for_network(network: Network) -> Result<()> { let mut state = NonFinalizedState::new(network); let finalized_state = FinalizedState::new(&Config::ephemeral(), network); + let fake_value_pool = ValueBalance::::fake_populated_pool(); + finalized_state.set_current_value_pool(fake_value_pool); + state.commit_new_chain(block1.prepare(), &finalized_state)?; state.commit_block(less_work_child.prepare(), &finalized_state)?; state.commit_block(more_work_child.prepare(), &finalized_state)?; diff --git a/zebra-state/src/service/tests.rs b/zebra-state/src/service/tests.rs index 205620a5..175fd344 100644 --- a/zebra-state/src/service/tests.rs +++ b/zebra-state/src/service/tests.rs @@ -1,10 +1,11 @@ -use std::{env, sync::Arc}; +use std::{convert::TryInto, env, sync::Arc}; use futures::stream::FuturesUnordered; use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt}; use zebra_chain::{ - block::Block, + block::{self, Block}, + fmt::SummaryDebug, parameters::{Network, NetworkUpgrade}, serialization::{ZcashDeserialize, ZcashDeserializeInto}, transaction, transparent, @@ -321,11 +322,11 @@ proptest! { /// 1. Generate a finalized chain and some non-finalized blocks. /// 2. Check that initially the value pool is empty. /// 3. Commit the finalized blocks and check that the value pool is updated accordingly. - /// 4. TODO: Commit the non-finalized blocks and check that the value pool is also updated + /// 4. Commit the non-finalized blocks and check that the value pool is also updated /// accordingly. #[test] fn value_pool_is_updated( - (network, finalized_blocks, _non_finalized_blocks) + (network, finalized_blocks, non_finalized_blocks) in continuous_empty_blocks_from_test_vectors(), ) { zebra_test::init(); @@ -333,17 +334,65 @@ proptest! { let (mut state_service, _) = StateService::new(Config::ephemeral(), network); prop_assert_eq!(state_service.disk.current_value_pool(), ValueBalance::zero()); + prop_assert_eq!( + state_service.mem.best_chain().map(|chain| chain.chain_value_pools).unwrap_or_else(ValueBalance::zero), + ValueBalance::zero() + ); - let mut expected_value_pool = Ok(ValueBalance::zero()); + // the slow start rate for the first few blocks, as in the spec + const SLOW_START_RATE: i64 = 62500; + // the expected transparent pool value, calculated using the slow start rate + let mut expected_transparent_pool = ValueBalance::zero(); + + let mut expected_finalized_value_pool = Ok(ValueBalance::zero()); for block in finalized_blocks { - let utxos = &block.new_outputs; - let block_value_pool = &block.block.chain_value_pool_change(utxos)?; - expected_value_pool += *block_value_pool; + // the genesis block has a zero-valued transparent output, + // which is not included in the UTXO set + if block.height > block::Height(0) { + let utxos = &block.new_outputs; + let block_value_pool = &block.block.chain_value_pool_change(utxos)?; + expected_finalized_value_pool += *block_value_pool; + } - state_service.queue_and_commit_finalized(block); + state_service.queue_and_commit_finalized(block.clone()); + + prop_assert_eq!( + state_service.disk.current_value_pool(), + expected_finalized_value_pool.clone()?.constrain()? + ); + + let transparent_value = SLOW_START_RATE * i64::from(block.height.0); + let transparent_value = transparent_value.try_into().unwrap(); + let transparent_value = ValueBalance::from_transparent_amount(transparent_value); + expected_transparent_pool = (expected_transparent_pool + transparent_value).unwrap(); + prop_assert_eq!( + state_service.disk.current_value_pool(), + expected_transparent_pool + ); } - prop_assert_eq!(state_service.disk.current_value_pool(), expected_value_pool?.constrain()?); + let mut expected_non_finalized_value_pool = Ok(expected_finalized_value_pool?); + for block in non_finalized_blocks { + let utxos = block.new_outputs.clone(); + let block_value_pool = &block.block.chain_value_pool_change(&transparent::utxos_from_ordered_utxos(utxos))?; + expected_non_finalized_value_pool += *block_value_pool; + + state_service.queue_and_commit_non_finalized(block.clone()); + + prop_assert_eq!( + state_service.mem.best_chain().unwrap().chain_value_pools, + expected_non_finalized_value_pool.clone()?.constrain()? + ); + + let transparent_value = SLOW_START_RATE * i64::from(block.height.0); + let transparent_value = transparent_value.try_into().unwrap(); + let transparent_value = ValueBalance::from_transparent_amount(transparent_value); + expected_transparent_pool = (expected_transparent_pool + transparent_value).unwrap(); + prop_assert_eq!( + state_service.mem.best_chain().unwrap().chain_value_pools, + expected_transparent_pool + ); + } } } @@ -352,8 +401,13 @@ proptest! { /// Selects either the mainnet or testnet chain test vector and randomly splits the chain in two /// lists of blocks. The first containing the blocks to be finalized (which always includes at /// least the genesis block) and the blocks to be stored in the non-finalized state. -fn continuous_empty_blocks_from_test_vectors( -) -> impl Strategy, Vec)> { +fn continuous_empty_blocks_from_test_vectors() -> impl Strategy< + Value = ( + Network, + SummaryDebug>, + SummaryDebug>, + ), +> { any::() .prop_flat_map(|network| { // Select the test vector based on the network @@ -389,6 +443,10 @@ fn continuous_empty_blocks_from_test_vectors( .map(|prepared_block| FinalizedBlock::from(prepared_block.block)) .collect(); - (network, finalized_blocks, non_finalized_blocks) + ( + network, + finalized_blocks.into(), + non_finalized_blocks.into(), + ) }) }