octo
/
Zebra
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change(state): Insert only the first tree in each series of identical trees into finalized state (#7266) 

* Pass ZebraDB to batch preparation

* Dedup the insertion of Sapling trees into database

* Dedup the insertion of Orchard trees into database

* Update snapshots

* Rename batch preparation of trees

* Simplify the naming of note commitment trees

* Correctly retrieve Sapling trees from fin state

* Correctly retrieve Orchard trees from fin state

* Simplify the naming of methods for Sprout trees

* Simplify the naming of methods for Sapling trees

* Simplify the naming of methods for Orchard trees

* Reduce disk reads by caching trees. (#7276)

* Bump the state minor version

* Reset the state patch version

* Simplify the preparation of genesis trees

* Store the roots of the trees of the genesis block

* Add the genesis roots to snapshots

* fix(test): Don't include shielded data in genesis blocks (#7302)

* fix(state): Fix marking format upgrades (#7304)

---------

Co-authored-by: Arya <aryasolhi@gmail.com>
This commit is contained in:
Marek 2023-08-09 02:32:27 +02:00 committed by GitHub
parent cce81b35c9
commit 57c9249141
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 255 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
zebra-chain/src/transaction/arbitrary.rs
View File

@ -109,20 +109,32 @@ impl Transaction {
option::of(any::<sapling::ShieldedData<sapling::PerSpendAnchor>>()), option::of(any::<sapling::ShieldedData<sapling::PerSpendAnchor>>()),
) )
.prop_map( .prop_map(
|( move |(
inputs,
outputs,
lock_time,
expiry_height,
joinsplit_data,
sapling_shielded_data,
)| Transaction::V4 {
inputs, inputs,
outputs, outputs,
lock_time, lock_time,
expiry_height, expiry_height,
joinsplit_data, joinsplit_data,
sapling_shielded_data, sapling_shielded_data,
)| {
Transaction::V4 {
inputs,
outputs,
lock_time,
expiry_height,
joinsplit_data: if ledger_state.height.is_min() {
// The genesis block should not contain any joinsplits.
None
} else {
joinsplit_data
},
sapling_shielded_data: if ledger_state.height.is_min() {
// The genesis block should not contain any shielded data.
None
} else {
sapling_shielded_data
},
}
}, },
) )
.boxed() .boxed()
@ -159,8 +171,18 @@ impl Transaction {
expiry_height, expiry_height,
inputs, inputs,
outputs, outputs,
sapling_shielded_data, sapling_shielded_data: if ledger_state.height.is_min() {
orchard_shielded_data, // The genesis block should not contain any shielded data.
None
} else {
sapling_shielded_data
},
orchard_shielded_data: if ledger_state.height.is_min() {
// The genesis block should not contain any shielded data.
None
} else {
orchard_shielded_data
},
} }
}, },
) )

4
zebra-state/src/constants.rs
View File

@ -48,11 +48,11 @@ pub(crate) const DATABASE_FORMAT_VERSION: u64 = 25;
/// - adding new column families, /// - adding new column families,
/// - changing the format of a column family in a compatible way, or /// - changing the format of a column family in a compatible way, or
/// - breaking changes with compatibility code in all supported Zebra versions. /// - breaking changes with compatibility code in all supported Zebra versions.
pub(crate) const DATABASE_FORMAT_MINOR_VERSION: u64 = 0; pub(crate) const DATABASE_FORMAT_MINOR_VERSION: u64 = 1;
/// The database format patch version, incremented each time the on-disk database format has a /// The database format patch version, incremented each time the on-disk database format has a
/// significant format compatibility fix. /// significant format compatibility fix.
pub(crate) const DATABASE_FORMAT_PATCH_VERSION: u64 = 2; pub(crate) const DATABASE_FORMAT_PATCH_VERSION: u64 = 0;
/// The name of the file containing the minor and patch database versions. /// The name of the file containing the minor and patch database versions.
/// ///

2
zebra-state/src/service/check/anchors.rs
View File

@ -152,7 +152,7 @@ fn fetch_sprout_final_treestates(
let input_tree = parent_chain let input_tree = parent_chain
.and_then(|chain| chain.sprout_trees_by_anchor.get(&joinsplit.anchor).cloned()) .and_then(|chain| chain.sprout_trees_by_anchor.get(&joinsplit.anchor).cloned())
.or_else(|| finalized_state.sprout_note_commitment_tree_by_anchor(&joinsplit.anchor)); .or_else(|| finalized_state.sprout_tree_by_anchor(&joinsplit.anchor));
if let Some(input_tree) = input_tree { if let Some(input_tree) = input_tree {
sprout_final_treestates.insert(joinsplit.anchor, input_tree); sprout_final_treestates.insert(joinsplit.anchor, input_tree);

12
zebra-state/src/service/check/tests/nullifier.rs
View File

@ -85,7 +85,7 @@ proptest! {
// randomly choose to commit the block to the finalized or non-finalized state // randomly choose to commit the block to the finalized or non-finalized state
if use_finalized_state { if use_finalized_state {
let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
// the block was committed // the block was committed
prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@ -352,7 +352,7 @@ proptest! {
// randomly choose to commit the next block to the finalized or non-finalized state // randomly choose to commit the next block to the finalized or non-finalized state
if duplicate_in_finalized_state { if duplicate_in_finalized_state {
let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
prop_assert!(commit_result.is_ok()); prop_assert!(commit_result.is_ok());
@ -452,7 +452,7 @@ proptest! {
// randomly choose to commit the block to the finalized or non-finalized state // randomly choose to commit the block to the finalized or non-finalized state
if use_finalized_state { if use_finalized_state {
let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(),None, "test");
prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
prop_assert!(commit_result.is_ok()); prop_assert!(commit_result.is_ok());
@ -634,7 +634,7 @@ proptest! {
// randomly choose to commit the next block to the finalized or non-finalized state // randomly choose to commit the next block to the finalized or non-finalized state
if duplicate_in_finalized_state { if duplicate_in_finalized_state {
let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(),None, "test");
prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
prop_assert!(commit_result.is_ok()); prop_assert!(commit_result.is_ok());
@ -732,7 +732,7 @@ proptest! {
// randomly choose to commit the block to the finalized or non-finalized state // randomly choose to commit the block to the finalized or non-finalized state
if use_finalized_state { if use_finalized_state {
let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
prop_assert!(commit_result.is_ok()); prop_assert!(commit_result.is_ok());
@ -923,7 +923,7 @@ proptest! {
// randomly choose to commit the next block to the finalized or non-finalized state // randomly choose to commit the next block to the finalized or non-finalized state
if duplicate_in_finalized_state { if duplicate_in_finalized_state {
let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
prop_assert!(commit_result.is_ok()); prop_assert!(commit_result.is_ok());

9
zebra-state/src/service/check/tests/utxo.rs
View File

@ -185,7 +185,7 @@ proptest! {
// randomly choose to commit the block to the finalized or non-finalized state // randomly choose to commit the block to the finalized or non-finalized state
if use_finalized_state { if use_finalized_state {
let block1 = CheckpointVerifiedBlock::from(Arc::new(block1)); let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
// the block was committed // the block was committed
prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@ -273,7 +273,7 @@ proptest! {
if use_finalized_state_spend { if use_finalized_state_spend {
let block2 = CheckpointVerifiedBlock::from(Arc::new(block2)); let block2 = CheckpointVerifiedBlock::from(Arc::new(block2));
let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(),None, "test");
// the block was committed // the block was committed
prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@ -612,7 +612,7 @@ proptest! {
if use_finalized_state_spend { if use_finalized_state_spend {
let block2 = CheckpointVerifiedBlock::from(block2.clone()); let block2 = CheckpointVerifiedBlock::from(block2.clone());
let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), "test"); let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), None, "test");
// the block was committed // the block was committed
prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@ -884,7 +884,8 @@ fn new_state_with_mainnet_transparent_data(
if use_finalized_state { if use_finalized_state {
let block1 = CheckpointVerifiedBlock::from(block1.clone()); let block1 = CheckpointVerifiedBlock::from(block1.clone());
let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test"); let commit_result =
finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
// the block was committed // the block was committed
assert_eq!( assert_eq!(

28
zebra-state/src/service/finalized_state.rs
View File

@ -20,7 +20,7 @@ use std::{
sync::Arc, sync::Arc,
}; };
use zebra_chain::{block, parameters::Network}; use zebra_chain::{block, parallel::tree::NoteCommitmentTrees, parameters::Network};
use crate::{ use crate::{
request::{FinalizableBlock, SemanticallyVerifiedBlockWithTrees, Treestate}, request::{FinalizableBlock, SemanticallyVerifiedBlockWithTrees, Treestate},
@ -168,10 +168,12 @@ impl FinalizedState {
pub fn commit_finalized( pub fn commit_finalized(
&mut self, &mut self,
ordered_block: QueuedCheckpointVerified, ordered_block: QueuedCheckpointVerified,
) -> Result<CheckpointVerifiedBlock, BoxError> { prev_note_commitment_trees: Option<NoteCommitmentTrees>,
) -> Result<(CheckpointVerifiedBlock, NoteCommitmentTrees), BoxError> {
let (checkpoint_verified, rsp_tx) = ordered_block; let (checkpoint_verified, rsp_tx) = ordered_block;
let result = self.commit_finalized_direct( let result = self.commit_finalized_direct(
checkpoint_verified.clone().into(), checkpoint_verified.clone().into(),
prev_note_commitment_trees,
"commit checkpoint-verified request", "commit checkpoint-verified request",
); );
@ -202,10 +204,10 @@ impl FinalizedState {
// and the block write task. // and the block write task.
let result = result.map_err(CloneError::from); let result = result.map_err(CloneError::from);
let _ = rsp_tx.send(result.clone().map_err(BoxError::from)); let _ = rsp_tx.send(result.clone().map(|(hash, _)| hash).map_err(BoxError::from));
result result
.map(|_hash| checkpoint_verified) .map(|(_hash, note_commitment_trees)| (checkpoint_verified, note_commitment_trees))
.map_err(BoxError::from) .map_err(BoxError::from)
} }
@ -226,9 +228,10 @@ impl FinalizedState {
pub fn commit_finalized_direct( pub fn commit_finalized_direct(
&mut self, &mut self,
finalizable_block: FinalizableBlock, finalizable_block: FinalizableBlock,
prev_note_commitment_trees: Option<NoteCommitmentTrees>,
source: &str, source: &str,
) -> Result<block::Hash, BoxError> { ) -> Result<(block::Hash, NoteCommitmentTrees), BoxError> {
let (height, hash, finalized) = match finalizable_block { let (height, hash, finalized, prev_note_commitment_trees) = match finalizable_block {
FinalizableBlock::Checkpoint { FinalizableBlock::Checkpoint {
checkpoint_verified, checkpoint_verified,
} => { } => {
@ -240,9 +243,11 @@ impl FinalizedState {
let block = checkpoint_verified.block.clone(); let block = checkpoint_verified.block.clone();
let mut history_tree = self.db.history_tree(); let mut history_tree = self.db.history_tree();
let mut note_commitment_trees = self.db.note_commitment_trees(); let prev_note_commitment_trees =
prev_note_commitment_trees.unwrap_or_else(|| self.db.note_commitment_trees());
// Update the note commitment trees. // Update the note commitment trees.
let mut note_commitment_trees = prev_note_commitment_trees.clone();
note_commitment_trees.update_trees_parallel(&block)?; note_commitment_trees.update_trees_parallel(&block)?;
// Check the block commitment if the history tree was not // Check the block commitment if the history tree was not
@ -287,6 +292,7 @@ impl FinalizedState {
history_tree, history_tree,
}, },
}, },
Some(prev_note_commitment_trees),
) )
} }
FinalizableBlock::Contextual { FinalizableBlock::Contextual {
@ -299,6 +305,7 @@ impl FinalizedState {
verified: contextually_verified.into(), verified: contextually_verified.into(),
treestate, treestate,
}, },
prev_note_commitment_trees,
), ),
}; };
@ -331,8 +338,11 @@ impl FinalizedState {
#[cfg(feature = "elasticsearch")] #[cfg(feature = "elasticsearch")]
let finalized_block = finalized.verified.block.clone(); let finalized_block = finalized.verified.block.clone();
let note_commitment_trees = finalized.treestate.note_commitment_trees.clone();
let result = self.db.write_block(finalized, self.network, source); let result =
self.db
.write_block(finalized, prev_note_commitment_trees, self.network, source);
if result.is_ok() { if result.is_ok() {
// Save blocks to elasticsearch if the feature is enabled. // Save blocks to elasticsearch if the feature is enabled.
@ -360,7 +370,7 @@ impl FinalizedState {
} }
} }
result result.map(|hash| (hash, note_commitment_trees))
} }
#[cfg(feature = "elasticsearch")] #[cfg(feature = "elasticsearch")]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
View File

@ -103,7 +103,7 @@ fn test_raw_rocksdb_column_families_with_network(network: Network) {
.expect("test data deserializes"); .expect("test data deserializes");
state state
.commit_finalized_direct(block.into(), "snapshot tests") .commit_finalized_direct(block.into(), None, "snapshot tests")
.expect("test block is valid"); .expect("test block is valid");
let mut settings = insta::Settings::clone_current(); let mut settings = insta::Settings::clone_current();

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_0.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000", v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
), ),
] ]

4
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_1.snap
View File

@ -5,10 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f", v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
), ),
] ]

8
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_2.snap
View File

@ -5,14 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
),
KV(
k: "000002",
v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f", v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_0.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000", v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
), ),
] ]

4
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_1.snap
View File

@ -5,10 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f", v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
), ),
] ]

8
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_2.snap
View File

@ -5,14 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
),
KV(
k: "000002",
v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f", v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_0.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000", v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
), ),
] ]

4
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_1.snap
View File

@ -5,10 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e", v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
), ),
] ]

8
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_2.snap
View File

@ -5,14 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
),
KV(
k: "000002",
v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e", v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@testnet_0.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000", v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
), ),
] ]

4
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@testnet_1.snap
View File

@ -5,10 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e", v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
), ),
] ]

8
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@testnet_2.snap
View File

@ -5,14 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000",
),
KV(
k: "000001",
v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
),
KV(
k: "000002",
v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e", v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sprout_note_commitment_tree_raw_data@mainnet_0.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000", v: "0001d7c612c817793191a1e68652121876d6b3bde40f4fa52bc314145ce6e5cdd259",
), ),
] ]

2
zebra-state/src/service/finalized_state/disk_format/tests/snapshots/sprout_note_commitment_tree_raw_data@testnet_0.snap
View File

@ -5,6 +5,6 @@ expression: cf_data
[ [
KV( KV(
k: "000000", k: "000000",
v: "0000", v: "0001d7c612c817793191a1e68652121876d6b3bde40f4fa52bc314145ce6e5cdd259",
), ),
] ]

22
zebra-state/src/service/finalized_state/disk_format/upgrade.rs
View File

@ -287,18 +287,18 @@ impl DbFormatChange {
upgrade_height = (upgrade_height + 1).expect("task exits before maximum height"); upgrade_height = (upgrade_height + 1).expect("task exits before maximum height");
} }
}
// At the end of each format upgrade, the database is marked as upgraded to that version. // At the end of each format upgrade, the database is marked as upgraded to that version.
// Upgrades can be run more than once if Zebra is restarted, so this is just a performance // Upgrades can be run more than once if Zebra is restarted, so this is just a performance
// optimisation. // optimisation.
info!( info!(
?initial_tip_height, ?initial_tip_height,
?newer_running_version, ?newer_running_version,
?older_disk_version, ?older_disk_version,
"marking database as upgraded" "marking database as upgraded"
); );
Self::mark_as_upgraded_to(&database_format_add_format_change_task, &config, network); Self::mark_as_upgraded_to(&database_format_add_format_change_task, &config, network);
}
// End of example format change. // End of example format change.

11
zebra-state/src/service/finalized_state/tests/prop.rs
View File

@ -29,12 +29,13 @@ fn blocks_with_v5_transactions() -> Result<()> {
// use `count` to minimize test failures, so they are easier to diagnose // use `count` to minimize test failures, so they are easier to diagnose
for block in chain.iter().take(count) { for block in chain.iter().take(count) {
let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone()); let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone());
let hash = state.commit_finalized_direct( let (hash, _) = state.commit_finalized_direct(
checkpoint_verified.into(), checkpoint_verified.into(),
None,
"blocks_with_v5_transactions test" "blocks_with_v5_transactions test"
); ).unwrap();
prop_assert_eq!(Some(height), state.finalized_tip_height()); prop_assert_eq!(Some(height), state.finalized_tip_height());
prop_assert_eq!(hash.unwrap(), block.hash); prop_assert_eq!(hash, block.hash);
height = Height(height.0 + 1); height = Height(height.0 + 1);
} }
}); });
@ -86,6 +87,7 @@ fn all_upgrades_and_wrong_commitments_with_fake_activation_heights() -> Result<(
let checkpoint_verified = CheckpointVerifiedBlock::from(block); let checkpoint_verified = CheckpointVerifiedBlock::from(block);
state.commit_finalized_direct( state.commit_finalized_direct(
checkpoint_verified.into(), checkpoint_verified.into(),
None,
"all_upgrades test" "all_upgrades test"
).expect_err("Must fail commitment check"); ).expect_err("Must fail commitment check");
failure_count += 1; failure_count += 1;
@ -93,8 +95,9 @@ fn all_upgrades_and_wrong_commitments_with_fake_activation_heights() -> Result<(
_ => {}, _ => {},
} }
let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone()); let checkpoint_verified = CheckpointVerifiedBlock::from(block.block.clone());
let hash = state.commit_finalized_direct( let (hash, _) = state.commit_finalized_direct(
checkpoint_verified.into(), checkpoint_verified.into(),
None,
"all_upgrades test" "all_upgrades test"
).unwrap(); ).unwrap();
prop_assert_eq!(Some(height), state.finalized_tip_height()); prop_assert_eq!(Some(height), state.finalized_tip_height());

104
zebra-state/src/service/finalized_state/zebra_db/block.rs
View File

@ -20,9 +20,11 @@ use zebra_chain::{
amount::NonNegative, amount::NonNegative,
block::{self, Block, Height}, block::{self, Block, Height},
orchard, orchard,
parallel::tree::NoteCommitmentTrees,
parameters::{Network, GENESIS_PREVIOUS_BLOCK_HASH}, parameters::{Network, GENESIS_PREVIOUS_BLOCK_HASH},
sapling, sapling,
serialization::TrustedPreallocate, serialization::TrustedPreallocate,
sprout,
transaction::{self, Transaction}, transaction::{self, Transaction},
transparent, transparent,
value_balance::ValueBalance, value_balance::ValueBalance,
@ -147,34 +149,28 @@ impl ZebraDb {
})) }))
} }
/// Returns the Sapling /// Returns the Sapling [`note commitment tree`](sapling::tree::NoteCommitmentTree) specified by
/// [`NoteCommitmentTree`](sapling::tree::NoteCommitmentTree) specified by a /// a hash or height, if it exists in the finalized state.
/// hash or height, if it exists in the finalized `db`.
#[allow(clippy::unwrap_in_result)] #[allow(clippy::unwrap_in_result)]
pub fn sapling_tree( pub fn sapling_tree_by_hash_or_height(
&self, &self,
hash_or_height: HashOrHeight, hash_or_height: HashOrHeight,
) -> Option<Arc<sapling::tree::NoteCommitmentTree>> { ) -> Option<Arc<sapling::tree::NoteCommitmentTree>> {
let height = hash_or_height.height_or_else(|hash| self.height(hash))?; let height = hash_or_height.height_or_else(|hash| self.height(hash))?;
let sapling_tree_handle = self.db.cf_handle("sapling_note_commitment_tree").unwrap(); self.sapling_tree_by_height(&height)
self.db.zs_get(&sapling_tree_handle, &height)
} }
/// Returns the Orchard /// Returns the Orchard [`note commitment tree`](orchard::tree::NoteCommitmentTree) specified by
/// [`NoteCommitmentTree`](orchard::tree::NoteCommitmentTree) specified by a /// a hash or height, if it exists in the finalized state.
/// hash or height, if it exists in the finalized `db`.
#[allow(clippy::unwrap_in_result)] #[allow(clippy::unwrap_in_result)]
pub fn orchard_tree( pub fn orchard_tree_by_hash_or_height(
&self, &self,
hash_or_height: HashOrHeight, hash_or_height: HashOrHeight,
) -> Option<Arc<orchard::tree::NoteCommitmentTree>> { ) -> Option<Arc<orchard::tree::NoteCommitmentTree>> {
let height = hash_or_height.height_or_else(|hash| self.height(hash))?; let height = hash_or_height.height_or_else(|hash| self.height(hash))?;
let orchard_tree_handle = self.db.cf_handle("orchard_note_commitment_tree").unwrap(); self.orchard_tree_by_height(&height)
self.db.zs_get(&orchard_tree_handle, &height)
} }
// Read tip block methods // Read tip block methods
@ -281,6 +277,7 @@ impl ZebraDb {
pub(in super::super) fn write_block( pub(in super::super) fn write_block(
&mut self, &mut self,
finalized: SemanticallyVerifiedBlockWithTrees, finalized: SemanticallyVerifiedBlockWithTrees,
prev_note_commitment_trees: Option<NoteCommitmentTrees>,
network: Network, network: Network,
source: &str, source: &str,
) -> Result<block::Hash, BoxError> { ) -> Result<block::Hash, BoxError> {
@ -375,13 +372,14 @@ impl ZebraDb {
// In case of errors, propagate and do not write the batch. // In case of errors, propagate and do not write the batch.
batch.prepare_block_batch( batch.prepare_block_batch(
&self.db, self,
&finalized, &finalized,
new_outputs_by_out_loc, new_outputs_by_out_loc,
spent_utxos_by_outpoint, spent_utxos_by_outpoint,
spent_utxos_by_out_loc, spent_utxos_by_out_loc,
address_balances, address_balances,
self.finalized_value_pool(), self.finalized_value_pool(),
prev_note_commitment_trees,
)?; )?;
self.db.write(batch)?; self.db.write(batch)?;
@ -426,14 +424,16 @@ impl DiskWriteBatch {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn prepare_block_batch( pub fn prepare_block_batch(
&mut self, &mut self,
db: &DiskDb, zebra_db: &ZebraDb,
finalized: &SemanticallyVerifiedBlockWithTrees, finalized: &SemanticallyVerifiedBlockWithTrees,
new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>, new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
spent_utxos_by_outpoint: HashMap<transparent::OutPoint, transparent::Utxo>, spent_utxos_by_outpoint: HashMap<transparent::OutPoint, transparent::Utxo>,
spent_utxos_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>, spent_utxos_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
address_balances: HashMap<transparent::Address, AddressBalanceLocation>, address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
value_pool: ValueBalance<NonNegative>, value_pool: ValueBalance<NonNegative>,
prev_note_commitment_trees: Option<NoteCommitmentTrees>,
) -> Result<(), BoxError> { ) -> Result<(), BoxError> {
let db = &zebra_db.db;
// Commit block and transaction data. // Commit block and transaction data.
// (Transaction indexes, note commitments, and UTXOs are committed later.) // (Transaction indexes, note commitments, and UTXOs are committed later.)
self.prepare_block_header_and_transaction_data_batch(db, &finalized.verified)?; self.prepare_block_header_and_transaction_data_batch(db, &finalized.verified)?;
@ -447,7 +447,7 @@ impl DiskWriteBatch {
// //
// By returning early, Zebra commits the genesis block and transaction data, // By returning early, Zebra commits the genesis block and transaction data,
// but it ignores the genesis UTXO and value pool updates. // but it ignores the genesis UTXO and value pool updates.
if self.prepare_genesis_batch(db, &finalized.verified) { if self.prepare_genesis_batch(db, finalized) {
return Ok(()); return Ok(());
} }
@ -462,7 +462,7 @@ impl DiskWriteBatch {
)?; )?;
self.prepare_shielded_transaction_batch(db, &finalized.verified)?; self.prepare_shielded_transaction_batch(db, &finalized.verified)?;
self.prepare_note_commitment_batch(db, finalized)?; self.prepare_trees_batch(zebra_db, finalized, prev_note_commitment_trees)?;
// Commit UTXOs and value pools // Commit UTXOs and value pools
self.prepare_chain_value_pools_batch( self.prepare_chain_value_pools_batch(
@ -538,29 +538,71 @@ impl DiskWriteBatch {
Ok(()) Ok(())
} }
/// If `finalized.block` is a genesis block, /// If `finalized.block` is a genesis block, prepares a database batch that finishes
/// prepare a database batch that finishes initializing the database, /// initializing the database, and returns `true` without actually writing anything.
/// and return `true` (without actually writing anything).
/// ///
/// Since the genesis block's transactions are skipped, /// Since the genesis block's transactions are skipped, the returned genesis batch should be
/// the returned genesis batch should be written to the database immediately. /// written to the database immediately.
/// ///
/// If `finalized.block` is not a genesis block, does nothing. /// If `finalized.block` is not a genesis block, does nothing.
/// ///
/// This method never returns an error. /// # Panics
///
/// If `finalized.block` is a genesis block, and a note commitment tree in `finalized` doesn't
/// match its corresponding empty tree.
pub fn prepare_genesis_batch( pub fn prepare_genesis_batch(
&mut self, &mut self,
db: &DiskDb, db: &DiskDb,
finalized: &SemanticallyVerifiedBlock, finalized: &SemanticallyVerifiedBlockWithTrees,
) -> bool { ) -> bool {
let SemanticallyVerifiedBlock { block, .. } = finalized; if finalized.verified.block.header.previous_block_hash == GENESIS_PREVIOUS_BLOCK_HASH {
assert_eq!(
*finalized.treestate.note_commitment_trees.sprout,
sprout::tree::NoteCommitmentTree::default(),
"The Sprout tree in the finalized block must match the empty Sprout tree."
);
assert_eq!(
*finalized.treestate.note_commitment_trees.sapling,
sapling::tree::NoteCommitmentTree::default(),
"The Sapling tree in the finalized block must match the empty Sapling tree."
);
assert_eq!(
*finalized.treestate.note_commitment_trees.orchard,
orchard::tree::NoteCommitmentTree::default(),
"The Orchard tree in the finalized block must match the empty Orchard tree."
);
if block.header.previous_block_hash == GENESIS_PREVIOUS_BLOCK_HASH { // We want to store the trees of the genesis block together with their roots, and since
self.prepare_genesis_note_commitment_tree_batch(db, finalized); // the trees cache the roots after their computation, we trigger the computation.
//
// At the time of writing this comment, the roots are precomputed before this function
// is called, so the roots should already be cached.
finalized.treestate.note_commitment_trees.sprout.root();
finalized.treestate.note_commitment_trees.sapling.root();
finalized.treestate.note_commitment_trees.orchard.root();
return true; // Insert the empty note commitment trees. Note that these can't be used too early
// (e.g. the Orchard tree before Nu5 activates) since the block validation will make
// sure only appropriate transactions are allowed in a block.
self.zs_insert(
&db.cf_handle("sprout_note_commitment_tree").unwrap(),
finalized.verified.height,
finalized.treestate.note_commitment_trees.sprout.clone(),
);
self.zs_insert(
&db.cf_handle("sapling_note_commitment_tree").unwrap(),
finalized.verified.height,
finalized.treestate.note_commitment_trees.sapling.clone(),
);
self.zs_insert(
&db.cf_handle("orchard_note_commitment_tree").unwrap(),
finalized.verified.height,
finalized.treestate.note_commitment_trees.orchard.clone(),
);
true
} else {
false
} }
false
} }
} }

18
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
View File

@ -197,7 +197,7 @@ fn test_block_and_transaction_data_with_network(network: Network) {
.expect("test data deserializes"); .expect("test data deserializes");
state state
.commit_finalized_direct(block.into(), "snapshot tests") .commit_finalized_direct(block.into(), None, "snapshot tests")
.expect("test block is valid"); .expect("test block is valid");
let mut settings = insta::Settings::clone_current(); let mut settings = insta::Settings::clone_current();
@ -220,10 +220,10 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
// //
// We only store the sprout tree for the tip by height, so we can't check sprout here. // We only store the sprout tree for the tip by height, so we can't check sprout here.
let sapling_tree = state let sapling_tree = state
.sapling_note_commitment_tree_by_height(&block::Height::MIN) .sapling_tree_by_height(&block::Height::MIN)
.expect("the genesis block in the database has a Sapling tree"); .expect("the genesis block in the database has a Sapling tree");
let orchard_tree = state let orchard_tree = state
.orchard_note_commitment_tree_by_height(&block::Height::MIN) .orchard_tree_by_height(&block::Height::MIN)
.expect("the genesis block in the database has an Orchard tree"); .expect("the genesis block in the database has an Orchard tree");
assert_eq!(*sapling_tree, sapling::tree::NoteCommitmentTree::default()); assert_eq!(*sapling_tree, sapling::tree::NoteCommitmentTree::default());
@ -243,13 +243,13 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
// Shielded // Shielded
let stored_sprout_trees = state.sprout_note_commitments_full_map(); let stored_sprout_trees = state.sprout_trees_full_map();
let mut stored_sapling_trees = Vec::new(); let mut stored_sapling_trees = Vec::new();
let mut stored_orchard_trees = Vec::new(); let mut stored_orchard_trees = Vec::new();
let sprout_tree_at_tip = state.sprout_note_commitment_tree(); let sprout_tree_at_tip = state.sprout_tree();
let sapling_tree_at_tip = state.sapling_note_commitment_tree(); let sapling_tree_at_tip = state.sapling_tree();
let orchard_tree_at_tip = state.orchard_note_commitment_tree(); let orchard_tree_at_tip = state.orchard_tree();
// Test the history tree. // Test the history tree.
// //
@ -278,10 +278,10 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) {
// //
// TODO: test the rest of the shielded data (anchors, nullifiers) // TODO: test the rest of the shielded data (anchors, nullifiers)
let sapling_tree_by_height = state let sapling_tree_by_height = state
.sapling_note_commitment_tree_by_height(&query_height) .sapling_tree_by_height(&query_height)
.expect("heights up to tip have Sapling trees"); .expect("heights up to tip have Sapling trees");
let orchard_tree_by_height = state let orchard_tree_by_height = state
.orchard_note_commitment_tree_by_height(&query_height) .orchard_tree_by_height(&query_height)
.expect("heights up to tip have Orchard trees"); .expect("heights up to tip have Orchard trees");
// We don't need to snapshot the heights, // We don't need to snapshot the heights,

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/orchard_trees@mainnet_1.snap
View File

@ -7,7 +7,9 @@ expression: stored_orchard_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Base(
bytes: (174, 41, 53, 241, 223, 216, 162, 74, 237, 124, 112, 223, 125, 227, 166, 104, 235, 122, 73, 177, 49, 152, 128, 221, 226, 187, 217, 3, 26, 229, 216, 47),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/orchard_trees@mainnet_2.snap
View File

@ -7,7 +7,9 @@ expression: stored_orchard_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Base(
bytes: (174, 41, 53, 241, 223, 216, 162, 74, 237, 124, 112, 223, 125, 227, 166, 104, 235, 122, 73, 177, 49, 152, 128, 221, 226, 187, 217, 3, 26, 229, 216, 47),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/orchard_trees@testnet_1.snap
View File

@ -7,7 +7,9 @@ expression: stored_orchard_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Base(
bytes: (174, 41, 53, 241, 223, 216, 162, 74, 237, 124, 112, 223, 125, 227, 166, 104, 235, 122, 73, 177, 49, 152, 128, 221, 226, 187, 217, 3, 26, 229, 216, 47),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/orchard_trees@testnet_2.snap
View File

@ -7,7 +7,9 @@ expression: stored_orchard_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Base(
bytes: (174, 41, 53, 241, 223, 216, 162, 74, 237, 124, 112, 223, 125, 227, 166, 104, 235, 122, 73, 177, 49, 152, 128, 221, 226, 187, 217, 3, 26, 229, 216, 47),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sapling_trees@mainnet_1.snap
View File

@ -7,7 +7,9 @@ expression: stored_sapling_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Fq(
bytes: (251, 194, 244, 48, 12, 1, 240, 183, 130, 13, 0, 227, 52, 124, 141, 164, 238, 97, 70, 116, 55, 108, 188, 69, 53, 157, 170, 84, 249, 181, 73, 62),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sapling_trees@mainnet_2.snap
View File

@ -7,7 +7,9 @@ expression: stored_sapling_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Fq(
bytes: (251, 194, 244, 48, 12, 1, 240, 183, 130, 13, 0, 227, 52, 124, 141, 164, 238, 97, 70, 116, 55, 108, 188, 69, 53, 157, 170, 84, 249, 181, 73, 62),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sapling_trees@testnet_1.snap
View File

@ -7,7 +7,9 @@ expression: stored_sapling_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Fq(
bytes: (251, 194, 244, 48, 12, 1, 240, 183, 130, 13, 0, 227, 52, 124, 141, 164, 238, 97, 70, 116, 55, 108, 188, 69, 53, 157, 170, 84, 249, 181, 73, 62),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

4
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sapling_trees@testnet_2.snap
View File

@ -7,7 +7,9 @@ expression: stored_sapling_trees
inner: Frontier( inner: Frontier(
frontier: None, frontier: None,
), ),
cached_root: None, cached_root: Some(Root(Fq(
bytes: (251, 194, 244, 48, 12, 1, 240, 183, 130, 13, 0, 227, 52, 124, 141, 164, 238, 97, 70, 116, 55, 108, 188, 69, 53, 157, 170, 84, 249, 181, 73, 62),
))),
)), )),
(Height(1), NoteCommitmentTree( (Height(1), NoteCommitmentTree(
inner: Frontier( inner: Frontier(

118
zebra-state/src/service/finalized_state/zebra_db/shielded.rs
View File

@ -70,7 +70,7 @@ impl ZebraDb {
/// Returns the Sprout note commitment tree of the finalized tip /// Returns the Sprout note commitment tree of the finalized tip
/// or the empty tree if the state is empty. /// or the empty tree if the state is empty.
pub fn sprout_note_commitment_tree(&self) -> Arc<sprout::tree::NoteCommitmentTree> { pub fn sprout_tree(&self) -> Arc<sprout::tree::NoteCommitmentTree> {
let height = match self.finalized_tip_height() { let height = match self.finalized_tip_height() {
Some(h) => h, Some(h) => h,
None => return Default::default(), None => return Default::default(),
@ -88,7 +88,7 @@ impl ZebraDb {
/// ///
/// This is used for interstitial tree building, which is unique to Sprout. /// This is used for interstitial tree building, which is unique to Sprout.
#[allow(clippy::unwrap_in_result)] #[allow(clippy::unwrap_in_result)]
pub fn sprout_note_commitment_tree_by_anchor( pub fn sprout_tree_by_anchor(
&self, &self,
sprout_anchor: &sprout::tree::Root, sprout_anchor: &sprout::tree::Root,
) -> Option<Arc<sprout::tree::NoteCommitmentTree>> { ) -> Option<Arc<sprout::tree::NoteCommitmentTree>> {
@ -103,7 +103,7 @@ impl ZebraDb {
/// ///
/// Calling this method can load a lot of data into RAM, and delay block commit transactions. /// Calling this method can load a lot of data into RAM, and delay block commit transactions.
#[allow(dead_code, clippy::unwrap_in_result)] #[allow(dead_code, clippy::unwrap_in_result)]
pub fn sprout_note_commitments_full_map( pub fn sprout_trees_full_map(
&self, &self,
) -> HashMap<sprout::tree::Root, Arc<sprout::tree::NoteCommitmentTree>> { ) -> HashMap<sprout::tree::Root, Arc<sprout::tree::NoteCommitmentTree>> {
let sprout_anchors_handle = self.db.cf_handle("sprout_anchors").unwrap(); let sprout_anchors_handle = self.db.cf_handle("sprout_anchors").unwrap();
@ -114,20 +114,20 @@ impl ZebraDb {
/// Returns the Sapling note commitment tree of the finalized tip /// Returns the Sapling note commitment tree of the finalized tip
/// or the empty tree if the state is empty. /// or the empty tree if the state is empty.
pub fn sapling_note_commitment_tree(&self) -> Arc<sapling::tree::NoteCommitmentTree> { pub fn sapling_tree(&self) -> Arc<sapling::tree::NoteCommitmentTree> {
let height = match self.finalized_tip_height() { let height = match self.finalized_tip_height() {
Some(h) => h, Some(h) => h,
None => return Default::default(), None => return Default::default(),
}; };
self.sapling_note_commitment_tree_by_height(&height) self.sapling_tree_by_height(&height)
.expect("Sapling note commitment tree must exist if there is a finalized tip") .expect("Sapling note commitment tree must exist if there is a finalized tip")
} }
/// Returns the Sapling note commitment tree matching the given block height, /// Returns the Sapling note commitment tree matching the given block height,
/// or `None` if the height is above the finalized tip. /// or `None` if the height is above the finalized tip.
#[allow(clippy::unwrap_in_result)] #[allow(clippy::unwrap_in_result)]
pub fn sapling_note_commitment_tree_by_height( pub fn sapling_tree_by_height(
&self, &self,
height: &Height, height: &Height,
) -> Option<Arc<sapling::tree::NoteCommitmentTree>> { ) -> Option<Arc<sapling::tree::NoteCommitmentTree>> {
@ -159,20 +159,20 @@ impl ZebraDb {
/// Returns the Orchard note commitment tree of the finalized tip /// Returns the Orchard note commitment tree of the finalized tip
/// or the empty tree if the state is empty. /// or the empty tree if the state is empty.
pub fn orchard_note_commitment_tree(&self) -> Arc<orchard::tree::NoteCommitmentTree> { pub fn orchard_tree(&self) -> Arc<orchard::tree::NoteCommitmentTree> {
let height = match self.finalized_tip_height() { let height = match self.finalized_tip_height() {
Some(h) => h, Some(h) => h,
None => return Default::default(), None => return Default::default(),
}; };
self.orchard_note_commitment_tree_by_height(&height) self.orchard_tree_by_height(&height)
.expect("Orchard note commitment tree must exist if there is a finalized tip") .expect("Orchard note commitment tree must exist if there is a finalized tip")
} }
/// Returns the Orchard note commitment tree matching the given block height, /// Returns the Orchard note commitment tree matching the given block height,
/// or `None` if the height is above the finalized tip. /// or `None` if the height is above the finalized tip.
#[allow(clippy::unwrap_in_result)] #[allow(clippy::unwrap_in_result)]
pub fn orchard_note_commitment_tree_by_height( pub fn orchard_tree_by_height(
&self, &self,
height: &Height, height: &Height,
) -> Option<Arc<orchard::tree::NoteCommitmentTree>> { ) -> Option<Arc<orchard::tree::NoteCommitmentTree>> {
@ -203,9 +203,9 @@ impl ZebraDb {
/// or the empty trees if the state is empty. /// or the empty trees if the state is empty.
pub fn note_commitment_trees(&self) -> NoteCommitmentTrees { pub fn note_commitment_trees(&self) -> NoteCommitmentTrees {
NoteCommitmentTrees { NoteCommitmentTrees {
sprout: self.sprout_note_commitment_tree(), sprout: self.sprout_tree(),
sapling: self.sapling_note_commitment_tree(), sapling: self.sapling_tree(),
orchard: self.orchard_note_commitment_tree(), orchard: self.orchard_tree(),
} }
} }
} }
@ -275,97 +275,67 @@ impl DiskWriteBatch {
/// ///
/// - Propagates any errors from updating the history tree /// - Propagates any errors from updating the history tree
#[allow(clippy::unwrap_in_result)] #[allow(clippy::unwrap_in_result)]
pub fn prepare_note_commitment_batch( pub fn prepare_trees_batch(
&mut self, &mut self,
db: &DiskDb, zebra_db: &ZebraDb,
finalized: &SemanticallyVerifiedBlockWithTrees, finalized: &SemanticallyVerifiedBlockWithTrees,
prev_note_commitment_trees: Option<NoteCommitmentTrees>,
) -> Result<(), BoxError> { ) -> Result<(), BoxError> {
let db = &zebra_db.db;
let sprout_anchors = db.cf_handle("sprout_anchors").unwrap(); let sprout_anchors = db.cf_handle("sprout_anchors").unwrap();
let sapling_anchors = db.cf_handle("sapling_anchors").unwrap(); let sapling_anchors = db.cf_handle("sapling_anchors").unwrap();
let orchard_anchors = db.cf_handle("orchard_anchors").unwrap(); let orchard_anchors = db.cf_handle("orchard_anchors").unwrap();
let sprout_note_commitment_tree_cf = db.cf_handle("sprout_note_commitment_tree").unwrap(); let sprout_tree_cf = db.cf_handle("sprout_note_commitment_tree").unwrap();
let sapling_note_commitment_tree_cf = db.cf_handle("sapling_note_commitment_tree").unwrap(); let sapling_tree_cf = db.cf_handle("sapling_note_commitment_tree").unwrap();
let orchard_note_commitment_tree_cf = db.cf_handle("orchard_note_commitment_tree").unwrap(); let orchard_tree_cf = db.cf_handle("orchard_note_commitment_tree").unwrap();
let height = finalized.verified.height; let height = finalized.verified.height;
let note_commitment_trees = finalized.treestate.note_commitment_trees.clone(); let trees = finalized.treestate.note_commitment_trees.clone();
// Use the cached values that were previously calculated in parallel. // Use the cached values that were previously calculated in parallel.
let sprout_root = note_commitment_trees.sprout.root(); let sprout_root = trees.sprout.root();
let sapling_root = note_commitment_trees.sapling.root(); let sapling_root = trees.sapling.root();
let orchard_root = note_commitment_trees.orchard.root(); let orchard_root = trees.orchard.root();
// Index the new anchors. // Index the new anchors.
// Note: if the root hasn't changed, we write the same value again. // Note: if the root hasn't changed, we write the same value again.
self.zs_insert(&sprout_anchors, sprout_root, &note_commitment_trees.sprout); self.zs_insert(&sprout_anchors, sprout_root, &trees.sprout);
self.zs_insert(&sapling_anchors, sapling_root, ()); self.zs_insert(&sapling_anchors, sapling_root, ());
self.zs_insert(&orchard_anchors, orchard_root, ()); self.zs_insert(&orchard_anchors, orchard_root, ());
// Delete the previously stored Sprout note commitment tree. // Delete the previously stored Sprout note commitment tree.
let current_tip_height = height - 1; let current_tip_height = height - 1;
if let Some(h) = current_tip_height { if let Some(h) = current_tip_height {
self.zs_delete(&sprout_note_commitment_tree_cf, h); self.zs_delete(&sprout_tree_cf, h);
} }
// TODO: if we ever need concurrent read-only access to the sprout tree, // TODO: if we ever need concurrent read-only access to the sprout tree,
// store it by `()`, not height. Otherwise, the ReadStateService could // store it by `()`, not height. Otherwise, the ReadStateService could
// access a height that was just deleted by a concurrent StateService // access a height that was just deleted by a concurrent StateService
// write. This requires a database version update. // write. This requires a database version update.
self.zs_insert( self.zs_insert(&sprout_tree_cf, height, trees.sprout);
&sprout_note_commitment_tree_cf,
height,
note_commitment_trees.sprout,
);
self.zs_insert( // Store the Sapling tree only if it is not already present at the previous height.
&sapling_note_commitment_tree_cf, if height.is_min()
height, || prev_note_commitment_trees
note_commitment_trees.sapling, .as_ref()
); .map_or_else(|| zebra_db.sapling_tree(), |trees| trees.sapling.clone())
!= trees.sapling
{
self.zs_insert(&sapling_tree_cf, height, trees.sapling);
}
self.zs_insert( // Store the Orchard tree only if it is not already present at the previous height.
&orchard_note_commitment_tree_cf, if height.is_min()
height, || prev_note_commitment_trees
note_commitment_trees.orchard, .map_or_else(|| zebra_db.orchard_tree(), |trees| trees.orchard)
); != trees.orchard
{
self.zs_insert(&orchard_tree_cf, height, trees.orchard);
}
self.prepare_history_batch(db, finalized) self.prepare_history_batch(db, finalized)
} }
/// Prepare a database batch containing the initial note commitment trees,
/// and return it (without actually writing anything).
///
/// This method never returns an error.
pub fn prepare_genesis_note_commitment_tree_batch(
&mut self,
db: &DiskDb,
finalized: &SemanticallyVerifiedBlock,
) {
let sprout_note_commitment_tree_cf = db.cf_handle("sprout_note_commitment_tree").unwrap();
let sapling_note_commitment_tree_cf = db.cf_handle("sapling_note_commitment_tree").unwrap();
let orchard_note_commitment_tree_cf = db.cf_handle("orchard_note_commitment_tree").unwrap();
let SemanticallyVerifiedBlock { height, .. } = finalized;
// Insert empty note commitment trees. Note that these can't be
// used too early (e.g. the Orchard tree before Nu5 activates)
// since the block validation will make sure only appropriate
// transactions are allowed in a block.
self.zs_insert(
&sprout_note_commitment_tree_cf,
height,
sprout::tree::NoteCommitmentTree::default(),
);
self.zs_insert(
&sapling_note_commitment_tree_cf,
height,
sapling::tree::NoteCommitmentTree::default(),
);
self.zs_insert(
&orchard_note_commitment_tree_cf,
height,
orchard::tree::NoteCommitmentTree::default(),
);
}
} }

6
zebra-state/src/service/non_finalized_state.rs
View File

@ -284,9 +284,9 @@ impl NonFinalizedState {
let chain = Chain::new( let chain = Chain::new(
self.network, self.network,
finalized_tip_height, finalized_tip_height,
finalized_state.sprout_note_commitment_tree(), finalized_state.sprout_tree(),
finalized_state.sapling_note_commitment_tree(), finalized_state.sapling_tree(),
finalized_state.orchard_note_commitment_tree(), finalized_state.orchard_tree(),
finalized_state.history_tree(), finalized_state.history_tree(),
finalized_state.finalized_value_pool(), finalized_state.finalized_value_pool(),
); );

4
zebra-state/src/service/read/tree.rs
View File

@ -38,7 +38,7 @@ where
// in memory, but `db` stores blocks on disk, with a memory cache.) // in memory, but `db` stores blocks on disk, with a memory cache.)
chain chain
.and_then(|chain| chain.as_ref().sapling_tree(hash_or_height)) .and_then(|chain| chain.as_ref().sapling_tree(hash_or_height))
.or_else(|| db.sapling_tree(hash_or_height)) .or_else(|| db.sapling_tree_by_hash_or_height(hash_or_height))
} }
/// Returns the Orchard /// Returns the Orchard
@ -59,7 +59,7 @@ where
// in memory, but `db` stores blocks on disk, with a memory cache.) // in memory, but `db` stores blocks on disk, with a memory cache.)
chain chain
.and_then(|chain| chain.as_ref().orchard_tree(hash_or_height)) .and_then(|chain| chain.as_ref().orchard_tree(hash_or_height))
.or_else(|| db.orchard_tree(hash_or_height)) .or_else(|| db.orchard_tree_by_hash_or_height(hash_or_height))
} }
#[cfg(feature = "getblocktemplate-rpcs")] #[cfg(feature = "getblocktemplate-rpcs")]

14
zebra-state/src/service/write.rs
View File

@ -140,6 +140,7 @@ pub fn write_blocks_from_channels(
non_finalized_state_sender: watch::Sender<NonFinalizedState>, non_finalized_state_sender: watch::Sender<NonFinalizedState>,
) { ) {
let mut last_zebra_mined_log_height = None; let mut last_zebra_mined_log_height = None;
let mut prev_finalized_note_commitment_trees = None;
// Write all the finalized blocks sent by the state, // Write all the finalized blocks sent by the state,
// until the state closes the finalized block channel's sender. // until the state closes the finalized block channel's sender.
@ -178,9 +179,12 @@ pub fn write_blocks_from_channels(
} }
// Try committing the block // Try committing the block
match finalized_state.commit_finalized(ordered_block) { match finalized_state
Ok(finalized) => { .commit_finalized(ordered_block, prev_finalized_note_commitment_trees.take())
{
Ok((finalized, note_commitment_trees)) => {
let tip_block = ChainTipBlock::from(finalized); let tip_block = ChainTipBlock::from(finalized);
prev_finalized_note_commitment_trees = Some(note_commitment_trees);
log_if_mined_by_zebra(&tip_block, &mut last_zebra_mined_log_height); log_if_mined_by_zebra(&tip_block, &mut last_zebra_mined_log_height);
@ -289,11 +293,11 @@ pub fn write_blocks_from_channels(
while non_finalized_state.best_chain_len() > MAX_BLOCK_REORG_HEIGHT { while non_finalized_state.best_chain_len() > MAX_BLOCK_REORG_HEIGHT {
tracing::trace!("finalizing block past the reorg limit"); tracing::trace!("finalizing block past the reorg limit");
let contextually_verified_with_trees = non_finalized_state.finalize(); let contextually_verified_with_trees = non_finalized_state.finalize();
finalized_state prev_finalized_note_commitment_trees = finalized_state
.commit_finalized_direct(contextually_verified_with_trees, "commit contextually-verified request") .commit_finalized_direct(contextually_verified_with_trees, prev_finalized_note_commitment_trees.take(), "commit contextually-verified request")
.expect( .expect(
"unexpected finalized block commit error: note commitment and history trees were already checked by the non-finalized state", "unexpected finalized block commit error: note commitment and history trees were already checked by the non-finalized state",
); ).1.into();
} }
// Update the metrics if semantic and contextual validation passes // Update the metrics if semantic and contextual validation passes

2
zebra-state/src/tests/setup.rs
View File

@ -107,7 +107,7 @@ pub(crate) fn new_state_with_mainnet_genesis(
let genesis = CheckpointVerifiedBlock::from(genesis); let genesis = CheckpointVerifiedBlock::from(genesis);
finalized_state finalized_state
.commit_finalized_direct(genesis.clone().into(), "test") .commit_finalized_direct(genesis.clone().into(), None, "test")
.expect("unexpected invalid genesis block test vector"); .expect("unexpected invalid genesis block test vector");
assert_eq!( assert_eq!(