diff --git a/Cargo.lock b/Cargo.lock index 63195c9f..2ac842f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5881,6 +5881,7 @@ dependencies = [ "once_cell", "proptest", "proptest-derive", + "rand 0.8.5", "rayon", "regex", "rlimit", diff --git a/tower-batch-control/Cargo.toml b/tower-batch-control/Cargo.toml index 12b4c265..87c7f1fa 100644 --- a/tower-batch-control/Cargo.toml +++ b/tower-batch-control/Cargo.toml @@ -39,7 +39,7 @@ color-eyre = "0.6.2" tinyvec = { version = "1.6.0", features = ["rustc_1_55"] } ed25519-zebra = "4.0.0" -rand = { version = "0.8.5", package = "rand" } +rand = "0.8.5" tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] } tokio-test = "0.4.2" diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index 7615d602..c23f8641 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -114,7 +114,7 @@ zcash_address = { version = "0.2.1", optional = true } proptest = { version = "1.2.0", optional = true } proptest-derive = { version = "0.3.0", optional = true } -rand = { version = "0.8.5", optional = true, package = "rand" } +rand = { version = "0.8.5", optional = true } rand_chacha = { version = "0.3.1", optional = true } tokio = { version = "1.28.2", features = ["tracing"], optional = true } @@ -137,7 +137,7 @@ tracing = "0.1.37" proptest = "1.2.0" proptest-derive = "0.3.0" -rand = { version = "0.8.5", package = "rand" } +rand = "0.8.5" rand_chacha = "0.3.1" tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] } diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml index 524bf70b..ddea3068 100644 --- a/zebra-consensus/Cargo.toml +++ b/zebra-consensus/Cargo.toml @@ -40,7 +40,7 @@ bellman = "0.14.0" bls12_381 = "0.8.0" halo2 = { package = "halo2_proofs", version = "0.3.0" } jubjub = "0.10.0" -rand = { version = "0.8.5", package = "rand" } +rand = "0.8.5" rayon = "1.7.0" chrono = { version = "0.4.26", default-features = false, features = ["clock", "std"] } diff --git a/zebra-network/Cargo.toml b/zebra-network/Cargo.toml index 7bb8e14f..73fb7c23 100644 --- a/zebra-network/Cargo.toml +++ b/zebra-network/Cargo.toml @@ -53,7 +53,7 @@ lazy_static = "1.4.0" num-integer = "0.1.45" ordered-map = "0.4.2" pin-project = "1.1.0" -rand = { version = "0.8.5", package = "rand" } +rand = "0.8.5" rayon = "1.7.0" regex = "1.8.4" serde = { version = "1.0.164", features = ["serde_derive"] } diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index e9e4aa0e..3b05d7ac 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -63,7 +63,7 @@ hex = { version = "0.4.3", features = ["serde"] } serde = { version = "1.0.164", features = ["serde_derive"] } # Experimental feature getblocktemplate-rpcs -rand = { version = "0.8.5", package = "rand", optional = true } +rand = { version = "0.8.5", optional = true } # ECC deps used by getblocktemplate-rpcs feature zcash_address = { version = "0.2.1", optional = true } diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index e4dcda07..143d772f 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -91,10 +91,11 @@ once_cell = "1.18.0" spandoc = "0.2.2" hex = { version = "0.4.3", features = ["serde"] } -insta = { version = "1.30.0", features = ["ron"] } +insta = { version = "1.30.0", features = ["ron", "redactions"] } proptest = "1.2.0" proptest-derive = "0.3.0" +rand = "0.8.5" halo2 = { package = "halo2_proofs", version = "0.3.0" } jubjub = "0.10.0" diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index f896b3aa..7e8ebe44 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -10,7 +10,14 @@ //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! be incremented each time the database format (column, serialization, etc) changes. -use std::{cmp::Ordering, fmt::Debug, path::Path, sync::Arc}; +use std::{ + cmp::Ordering, + collections::{BTreeMap, HashMap}, + fmt::Debug, + ops::RangeBounds, + path::Path, + sync::Arc, +}; use itertools::Itertools; use rlimit::increase_nofile_limit; @@ -146,6 +153,7 @@ impl WriteDisk for DiskWriteBatch { /// defined format // // TODO: just implement these methods directly on DiskDb +// move this trait, its methods, and support methods to another module pub trait ReadDisk { /// Returns true if a rocksdb column family `cf` does not contain any entries. fn zs_is_empty(&self, cf: &C) -> bool @@ -202,6 +210,26 @@ pub trait ReadDisk { C: rocksdb::AsColumnFamilyRef, K: IntoDisk + FromDisk, V: FromDisk; + + /// Returns the keys and values in `cf` in `range`, in an ordered `BTreeMap`. + /// + /// Holding this iterator open might delay block commit transactions. + fn zs_items_in_range_ordered(&self, cf: &C, range: R) -> BTreeMap + where + C: rocksdb::AsColumnFamilyRef, + K: IntoDisk + FromDisk + Ord, + V: FromDisk, + R: RangeBounds; + + /// Returns the keys and values in `cf` in `range`, in an unordered `HashMap`. + /// + /// Holding this iterator open might delay block commit transactions. + fn zs_items_in_range_unordered(&self, cf: &C, range: R) -> HashMap + where + C: rocksdb::AsColumnFamilyRef, + K: IntoDisk + FromDisk + Eq + std::hash::Hash, + V: FromDisk, + R: RangeBounds; } impl PartialEq for DiskDb { @@ -342,6 +370,26 @@ impl ReadDisk for DiskDb { }) .expect("unexpected database failure") } + + fn zs_items_in_range_ordered(&self, cf: &C, range: R) -> BTreeMap + where + C: rocksdb::AsColumnFamilyRef, + K: IntoDisk + FromDisk + Ord, + V: FromDisk, + R: RangeBounds, + { + self.zs_range_iter(cf, range).collect() + } + + fn zs_items_in_range_unordered(&self, cf: &C, range: R) -> HashMap + where + C: rocksdb::AsColumnFamilyRef, + K: IntoDisk + FromDisk + Eq + std::hash::Hash, + V: FromDisk, + R: RangeBounds, + { + self.zs_range_iter(cf, range).collect() + } } impl DiskWriteBatch { @@ -366,6 +414,58 @@ impl DiskWriteBatch { } impl DiskDb { + /// Returns an iterator over the items in `cf` in `range`. + /// + /// Holding this iterator open might delay block commit transactions. + fn zs_range_iter(&self, cf: &C, range: R) -> impl Iterator + '_ + where + C: rocksdb::AsColumnFamilyRef, + K: IntoDisk + FromDisk, + V: FromDisk, + R: RangeBounds, + { + use std::ops::Bound::{self, *}; + + // Replace with map() when it stabilises: + // https://github.com/rust-lang/rust/issues/86026 + let map_to_vec = |bound: Bound<&K>| -> Bound> { + match bound { + Unbounded => Unbounded, + Included(x) => Included(x.as_bytes().as_ref().to_vec()), + Excluded(x) => Excluded(x.as_bytes().as_ref().to_vec()), + } + }; + + let start_bound = map_to_vec(range.start_bound()); + let end_bound = map_to_vec(range.end_bound()); + let range = (start_bound.clone(), end_bound); + + let start_bound_vec = + if let Included(ref start_bound) | Excluded(ref start_bound) = start_bound { + start_bound.clone() + } else { + // Actually unused + Vec::new() + }; + + let start_mode = if matches!(start_bound, Unbounded) { + // Unbounded iterators start at the first item + rocksdb::IteratorMode::Start + } else { + rocksdb::IteratorMode::From(start_bound_vec.as_slice(), rocksdb::Direction::Forward) + }; + + // Reading multiple items from iterators has caused database hangs, + // in previous RocksDB versions + self.db + .iterator_cf(cf, start_mode) + .map(|result| result.expect("unexpected database failure")) + .map(|(key, value)| (key.to_vec(), value)) + // Handle Excluded start and the end bound + .filter(move |(key, _value)| range.contains(key)) + .map(|(key, value)| (K::from_bytes(key), V::from_bytes(value))) + } + /// The ideal open file limit for Zebra const IDEAL_OPEN_FILE_LIMIT: u64 = 1024; diff --git a/zebra-state/src/service/finalized_state/disk_format/shielded.rs b/zebra-state/src/service/finalized_state/disk_format/shielded.rs index 8836549c..3b136236 100644 --- a/zebra-state/src/service/finalized_state/disk_format/shielded.rs +++ b/zebra-state/src/service/finalized_state/disk_format/shielded.rs @@ -44,6 +44,13 @@ impl IntoDisk for sprout::tree::Root { } } +impl FromDisk for sprout::tree::Root { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + let array: [u8; 32] = bytes.as_ref().try_into().unwrap(); + array.into() + } +} + impl IntoDisk for sapling::tree::Root { type Bytes = [u8; 32]; @@ -52,6 +59,13 @@ impl IntoDisk for sapling::tree::Root { } } +impl FromDisk for sapling::tree::Root { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + let array: [u8; 32] = bytes.as_ref().try_into().unwrap(); + array.try_into().expect("finalized data must be valid") + } +} + impl IntoDisk for orchard::tree::Root { type Bytes = [u8; 32]; @@ -60,6 +74,13 @@ impl IntoDisk for orchard::tree::Root { } } +impl FromDisk for orchard::tree::Root { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + let array: [u8; 32] = bytes.as_ref().try_into().unwrap(); + array.try_into().expect("finalized data must be valid") + } +} + // The following implementations for the note commitment trees use `serde` and // `bincode` because currently the inner Merkle tree frontier (from // `incrementalmerkletree`) only supports `serde` for serialization. `bincode` diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs b/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs index 8c5edfa0..6b726108 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs @@ -279,6 +279,13 @@ fn serialized_sprout_tree_root_equal() { ); } +#[test] +fn roundtrip_sprout_tree_root() { + let _init_guard = zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary // Sapling @@ -347,6 +354,13 @@ fn serialized_sapling_tree_root_equal() { ); } +#[test] +fn roundtrip_sapling_tree_root() { + let _init_guard = zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary // Orchard @@ -415,6 +429,13 @@ fn serialized_orchard_tree_root_equal() { ); } +#[test] +fn roundtrip_orchard_tree_root() { + let _init_guard = zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + // TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary // Chain diff --git a/zebra-state/src/service/finalized_state/tests/vectors.rs b/zebra-state/src/service/finalized_state/tests/vectors.rs index 73f787ac..98975646 100644 --- a/zebra-state/src/service/finalized_state/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/tests/vectors.rs @@ -1,11 +1,142 @@ //! Fixed test vectors for the finalized state. +//! These tests contain snapshots of the note commitment tree serialization format. +//! +//! We don't need to check empty trees, because the database format snapshot tests +//! use empty trees. use halo2::pasta::{group::ff::PrimeField, pallas}; use hex::FromHex; +use rand::random; + +use zebra_chain::{orchard, sapling, sprout}; use crate::service::finalized_state::disk_format::{FromDisk, IntoDisk}; -use zebra_chain::{orchard, sapling}; +/// Check that the sprout tree database serialization format has not changed. +#[test] +fn sprout_note_commitment_tree_serialization() { + let _init_guard = zebra_test::init(); + + let mut incremental_tree = sprout::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/sprout/tests/test_vectors.rs + let hex_commitments = [ + "62fdad9bfbf17c38ea626a9c9b8af8a748e6b4367c8494caf0ca592999e8b6ba", + "68eb35bc5e1ddb80a761718e63a1ecf4d4977ae22cc19fa732b85515b2a4c943", + "836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb", + ]; + + for (idx, cm_hex) in hex_commitments.iter().enumerate() { + let bytes = <[u8; 32]>::from_hex(cm_hex).unwrap(); + + let cm = sprout::NoteCommitment::from(bytes); + incremental_tree.append(cm).unwrap(); + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); + // Cache the root half of the time to make sure it works in both cases + let _ = incremental_tree.root(); + } + } + + // Make sure the last root is cached + let _ = incremental_tree.root(); + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "010200836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb019f5b2b1e4bf7e7318d0a1f417ca6bca36077025b3d11e074b94cd55ce9f3861801c45297124f50dcd3f78eed017afd1e30764cd74cdf0a57751978270fd0721359"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = sprout::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} + +/// Check that the sprout tree database serialization format has not changed for one commitment. +#[test] +fn sprout_note_commitment_tree_serialization_one() { + let _init_guard = zebra_test::init(); + + let mut incremental_tree = sprout::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/sprout/tests/test_vectors.rs + let hex_commitments = ["836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb"]; + + for (idx, cm_hex) in hex_commitments.iter().enumerate() { + let bytes = <[u8; 32]>::from_hex(cm_hex).unwrap(); + + let cm = sprout::NoteCommitment::from(bytes); + incremental_tree.append(cm).unwrap(); + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); + // Cache the root half of the time to make sure it works in both cases + let _ = incremental_tree.root(); + } + } + + // Make sure the last root is cached + let _ = incremental_tree.root(); + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "010000836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb000193e5f97ce1d5d94d0c6e1b66a4a262c9ae89e56e28f3f6e4a557b6fb70e173a8"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = sprout::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} + +/// Check that the sprout tree database serialization format has not changed when the number of +/// commitments is a power of two. +/// +/// Some trees have special handling for even numbers of roots, or powers of two, +/// so we also check that case. +#[test] +fn sprout_note_commitment_tree_serialization_pow2() { + let _init_guard = zebra_test::init(); + + let mut incremental_tree = sprout::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/sprout/tests/test_vectors.rs + let hex_commitments = [ + "62fdad9bfbf17c38ea626a9c9b8af8a748e6b4367c8494caf0ca592999e8b6ba", + "68eb35bc5e1ddb80a761718e63a1ecf4d4977ae22cc19fa732b85515b2a4c943", + "836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb", + "92498a8295ea36d593eaee7cb8b55be3a3e37b8185d3807693184054cd574ae4", + ]; + + for (idx, cm_hex) in hex_commitments.iter().enumerate() { + let bytes = <[u8; 32]>::from_hex(cm_hex).unwrap(); + + let cm = sprout::NoteCommitment::from(bytes); + incremental_tree.append(cm).unwrap(); + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); + // Cache the root half of the time to make sure it works in both cases + let _ = incremental_tree.root(); + } + } + + // Make sure the last root is cached + let _ = incremental_tree.root(); + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "010301836045484077cf6390184ea7cd48b460e2d0f22b2293b69633bb152314a692fb92498a8295ea36d593eaee7cb8b55be3a3e37b8185d3807693184054cd574ae4019f5b2b1e4bf7e7318d0a1f417ca6bca36077025b3d11e074b94cd55ce9f3861801b61f588fcba9cea79e94376adae1c49583f716d2f20367141f1369a235b95c98"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = sprout::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} + +/// Check that the sapling tree database serialization format has not changed. #[test] fn sapling_note_commitment_tree_serialization() { let _init_guard = zebra_test::init(); @@ -24,7 +155,8 @@ fn sapling_note_commitment_tree_serialization() { let cm_u = jubjub::Fq::from_bytes(&bytes).unwrap(); incremental_tree.append(cm_u).unwrap(); - if idx % 2 == 0 { + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); // Cache the root half of the time to make sure it works in both cases let _ = incremental_tree.root(); } @@ -45,6 +177,94 @@ fn sapling_note_commitment_tree_serialization() { assert_eq!(incremental_tree.root(), deserialized_tree.root()); } +/// Check that the sapling tree database serialization format has not changed for one commitment. +#[test] +fn sapling_note_commitment_tree_serialization_one() { + let _init_guard = zebra_test::init(); + + let mut incremental_tree = sapling::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/sapling/tests/test_vectors.rs + let hex_commitments = ["225747f3b5d5dab4e5a424f81f85c904ff43286e0f3fd07ef0b8c6a627b11458"]; + + for (idx, cm_u_hex) in hex_commitments.iter().enumerate() { + let bytes = <[u8; 32]>::from_hex(cm_u_hex).unwrap(); + + let cm_u = jubjub::Fq::from_bytes(&bytes).unwrap(); + incremental_tree.append(cm_u).unwrap(); + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); + // Cache the root half of the time to make sure it works in both cases + let _ = incremental_tree.root(); + } + } + + // Make sure the last root is cached + let _ = incremental_tree.root(); + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "010000225747f3b5d5dab4e5a424f81f85c904ff43286e0f3fd07ef0b8c6a627b1145800012c60c7de033d7539d123fb275011edfe08d57431676981d162c816372063bc71"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = sapling::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} + +/// Check that the sapling tree database serialization format has not changed when the number of +/// commitments is a power of two. +/// +/// Some trees have special handling for even numbers of roots, or powers of two, +/// so we also check that case. +#[test] +fn sapling_note_commitment_tree_serialization_pow2() { + let _init_guard = zebra_test::init(); + + let mut incremental_tree = sapling::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/sapling/tests/test_vectors.rs + let hex_commitments = [ + "3a27fed5dbbc475d3880360e38638c882fd9b273b618fc433106896083f77446", + "c7ca8f7df8fd997931d33985d935ee2d696856cc09cc516d419ea6365f163008", + "f0fa37e8063b139d342246142fc48e7c0c50d0a62c97768589e06466742c3702", + "e6d4d7685894d01b32f7e081ab188930be6c2b9f76d6847b7f382e3dddd7c608", + "8cebb73be883466d18d3b0c06990520e80b936440a2c9fd184d92a1f06c4e826", + "22fab8bcdb88154dbf5877ad1e2d7f1b541bc8a5ec1b52266095381339c27c03", + "f43e3aac61e5a753062d4d0508c26ceaf5e4c0c58ba3c956e104b5d2cf67c41c", + "3a3661bc12b72646c94bc6c92796e81953985ee62d80a9ec3645a9a95740ac15", + ]; + + for (idx, cm_u_hex) in hex_commitments.iter().enumerate() { + let bytes = <[u8; 32]>::from_hex(cm_u_hex).unwrap(); + + let cm_u = jubjub::Fq::from_bytes(&bytes).unwrap(); + incremental_tree.append(cm_u).unwrap(); + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); + // Cache the root half of the time to make sure it works in both cases + let _ = incremental_tree.root(); + } + } + + // Make sure the last root is cached + let _ = incremental_tree.root(); + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "010701f43e3aac61e5a753062d4d0508c26ceaf5e4c0c58ba3c956e104b5d2cf67c41c3a3661bc12b72646c94bc6c92796e81953985ee62d80a9ec3645a9a95740ac15025991131c5c25911b35fcea2a8343e2dfd7a4d5b45493390e0cb184394d91c349002df68503da9247dfde6585cb8c9fa94897cf21735f8fc1b32116ef474de05c01d23765f3d90dfd97817ed6d995bd253d85967f77b9f1eaef6ecbcb0ef6796812"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = sapling::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} + +/// Check that the orchard tree database serialization format has not changed. #[test] fn orchard_note_commitment_tree_serialization() { let _init_guard = zebra_test::init(); @@ -73,7 +293,8 @@ fn orchard_note_commitment_tree_serialization() { for (idx, cm_x_bytes) in commitments.iter().enumerate() { let cm_x = pallas::Base::from_repr(*cm_x_bytes).unwrap(); incremental_tree.append(cm_x).unwrap(); - if idx % 2 == 0 { + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); // Cache the root half of the time to make sure it works in both cases let _ = incremental_tree.root(); } @@ -93,3 +314,92 @@ fn orchard_note_commitment_tree_serialization() { assert_eq!(incremental_tree.root(), deserialized_tree.root()); } + +/// Check that the orchard tree database serialization format has not changed for one commitment. +#[test] +fn orchard_note_commitment_tree_serialization_one() { + let _init_guard = zebra_test::init(); + + let mut incremental_tree = orchard::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/orchard/tests/tree.rs + let commitments = [[ + 0x68, 0x13, 0x5c, 0xf4, 0x99, 0x33, 0x22, 0x90, 0x99, 0xa4, 0x4e, 0xc9, 0x9a, 0x75, 0xe1, + 0xe1, 0xcb, 0x46, 0x40, 0xf9, 0xb5, 0xbd, 0xec, 0x6b, 0x32, 0x23, 0x85, 0x6f, 0xea, 0x16, + 0x39, 0x0a, + ]]; + + for (idx, cm_x_bytes) in commitments.iter().enumerate() { + let cm_x = pallas::Base::from_repr(*cm_x_bytes).unwrap(); + incremental_tree.append(cm_x).unwrap(); + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); + // Cache the root half of the time to make sure it works in both cases + let _ = incremental_tree.root(); + } + } + + // Make sure the last root is cached + let _ = incremental_tree.root(); + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "01000068135cf49933229099a44ec99a75e1e1cb4640f9b5bdec6b3223856fea16390a000178afd4da59c541e9c2f317f9aff654f1fb38d14dc99431cbbfa93601c7068117"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = orchard::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} + +/// Check that the orchard tree database serialization format has not changed when the number of +/// commitments is a power of two. +/// +/// Some trees have special handling for even numbers of roots, or powers of two, +/// so we also check that case. +#[test] +fn orchard_note_commitment_tree_serialization_pow2() { + let _init_guard = zebra_test::init(); + + let mut incremental_tree = orchard::tree::NoteCommitmentTree::default(); + + // Some commitments from zebra-chain/src/orchard/tests/tree.rs + let commitments = [ + [ + 0x78, 0x31, 0x50, 0x08, 0xfb, 0x29, 0x98, 0xb4, 0x30, 0xa5, 0x73, 0x1d, 0x67, 0x26, + 0x20, 0x7d, 0xc0, 0xf0, 0xec, 0x81, 0xea, 0x64, 0xaf, 0x5c, 0xf6, 0x12, 0x95, 0x69, + 0x01, 0xe7, 0x2f, 0x0e, + ], + [ + 0xee, 0x94, 0x88, 0x05, 0x3a, 0x30, 0xc5, 0x96, 0xb4, 0x30, 0x14, 0x10, 0x5d, 0x34, + 0x77, 0xe6, 0xf5, 0x78, 0xc8, 0x92, 0x40, 0xd1, 0xd1, 0xee, 0x17, 0x43, 0xb7, 0x7b, + 0xb6, 0xad, 0xc4, 0x0a, + ], + ]; + + for (idx, cm_x_bytes) in commitments.iter().enumerate() { + let cm_x = pallas::Base::from_repr(*cm_x_bytes).unwrap(); + incremental_tree.append(cm_x).unwrap(); + if random() { + info!(?idx, "randomly caching root for note commitment tree index"); + // Cache the root half of the time to make sure it works in both cases + let _ = incremental_tree.root(); + } + } + + // Make sure the last root is cached + let _ = incremental_tree.root(); + + // This test vector was generated by the code itself. + // The purpose of this test is to make sure the serialization format does + // not change by accident. + let expected_serialized_tree_hex = "01010178315008fb2998b430a5731d6726207dc0f0ec81ea64af5cf612956901e72f0eee9488053a30c596b43014105d3477e6f578c89240d1d1ee1743b77bb6adc40a0001d3d525931005e45f5a29bc82524e871e5ee1b6d77839deb741a6e50cd99fdf1a"; + let serialized_tree = incremental_tree.as_bytes(); + assert_eq!(hex::encode(&serialized_tree), expected_serialized_tree_hex); + + let deserialized_tree = orchard::tree::NoteCommitmentTree::from_bytes(serialized_tree); + + assert_eq!(incremental_tree.root(), deserialized_tree.root()); +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs index 8ce1e67e..9c19f29e 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs @@ -217,6 +217,8 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { if let Some((max_height, tip_block_hash)) = tip { // Check that the database returns empty note commitment trees for the // genesis block. + // + // We only store the sprout tree for the tip by height, so we can't check sprout here. let sapling_tree = state .sapling_note_commitment_tree_by_height(&block::Height::MIN) .expect("the genesis block in the database has a Sapling tree"); @@ -241,9 +243,11 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { // Shielded + let stored_sprout_trees = state.sprout_note_commitments_full_map(); let mut stored_sapling_trees = Vec::new(); let mut stored_orchard_trees = Vec::new(); + let sprout_tree_at_tip = state.sprout_note_commitment_tree(); let sapling_tree_at_tip = state.sapling_note_commitment_tree(); let orchard_tree_at_tip = state.orchard_note_commitment_tree(); @@ -268,9 +272,11 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { .block(query_height.into()) .expect("heights up to tip have blocks"); - // Check the sapling and orchard note commitment trees. + // Check the shielded note commitment trees. // - // TODO: test the rest of the shielded data (anchors, nullifiers, sprout) + // We only store the sprout tree for the tip by height, so we can't check sprout here. + // + // TODO: test the rest of the shielded data (anchors, nullifiers) let sapling_tree_by_height = state .sapling_note_commitment_tree_by_height(&query_height) .expect("heights up to tip have Sapling trees"); @@ -297,6 +303,18 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { if query_height == max_height { assert_eq!(stored_block_hash, tip_block_hash); + // We only store the sprout tree for the tip by height, + // so the sprout check is less strict. + // We enforce the tip tree order by snapshotting it as well. + if let Some(stored_tree) = stored_sprout_trees.get(&sprout_tree_at_tip.root()) { + assert_eq!( + &sprout_tree_at_tip, stored_tree, + "unexpected missing sprout tip tree:\n\ + all trees: {stored_sprout_trees:?}" + ); + } else { + assert_eq!(sprout_tree_at_tip, Default::default()); + } assert_eq!(sapling_tree_at_tip, sapling_tree_by_height); assert_eq!(orchard_tree_at_tip, orchard_tree_by_height); @@ -427,6 +445,14 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { // These snapshots will change if the trees do not have cached roots. // But we expect them to always have cached roots, // because those roots are used to populate the anchor column families. + insta::assert_ron_snapshot!("sprout_tree_at_tip", sprout_tree_at_tip); + insta::assert_ron_snapshot!( + "sprout_trees", + stored_sprout_trees, + { + "." => insta::sorted_redaction() + } + ); insta::assert_ron_snapshot!("sapling_trees", stored_sapling_trees); insta::assert_ron_snapshot!("orchard_trees", stored_orchard_trees); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_0.snap new file mode 100644 index 00000000..b1835399 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_0.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: sprout_tree_at_tip +--- +NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), +) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_1.snap new file mode 100644 index 00000000..b1835399 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_1.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: sprout_tree_at_tip +--- +NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), +) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_2.snap new file mode 100644 index 00000000..b1835399 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@mainnet_2.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: sprout_tree_at_tip +--- +NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), +) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_0.snap new file mode 100644 index 00000000..b1835399 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_0.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: sprout_tree_at_tip +--- +NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), +) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_1.snap new file mode 100644 index 00000000..b1835399 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_1.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: sprout_tree_at_tip +--- +NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), +) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_2.snap new file mode 100644 index 00000000..b1835399 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_tree_at_tip@testnet_2.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: sprout_tree_at_tip +--- +NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), +) diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_0.snap new file mode 100644 index 00000000..fc004edd --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_0.snap @@ -0,0 +1,5 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_sprout_trees +--- +{} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_1.snap new file mode 100644 index 00000000..438e0809 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_1.snap @@ -0,0 +1,12 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_sprout_trees +--- +{ + Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), + ), +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_2.snap new file mode 100644 index 00000000..438e0809 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@mainnet_2.snap @@ -0,0 +1,12 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_sprout_trees +--- +{ + Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), + ), +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_0.snap new file mode 100644 index 00000000..fc004edd --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_0.snap @@ -0,0 +1,5 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_sprout_trees +--- +{} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_1.snap new file mode 100644 index 00000000..438e0809 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_1.snap @@ -0,0 +1,12 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_sprout_trees +--- +{ + Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), + ), +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_2.snap new file mode 100644 index 00000000..438e0809 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/sprout_trees@testnet_2.snap @@ -0,0 +1,12 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_sprout_trees +--- +{ + Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89)): NoteCommitmentTree( + inner: Frontier( + frontier: None, + ), + cached_root: Some(Root((215, 198, 18, 200, 23, 121, 49, 145, 161, 230, 134, 82, 18, 24, 118, 214, 179, 189, 228, 15, 79, 165, 43, 195, 20, 20, 92, 230, 229, 205, 210, 89))), + ), +} diff --git a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs index b5bfe260..68a75ae1 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs @@ -12,7 +12,7 @@ //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! be incremented each time the database format (column, serialization, etc) changes. -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use zebra_chain::{ block::Height, orchard, parallel::tree::NoteCommitmentTrees, sapling, sprout, @@ -99,6 +99,19 @@ impl ZebraDb { .map(Arc::new) } + /// Returns all the Sprout note commitment trees in the database. + /// + /// Calling this method can load a lot of data into RAM, and delay block commit transactions. + #[allow(dead_code, clippy::unwrap_in_result)] + pub fn sprout_note_commitments_full_map( + &self, + ) -> HashMap> { + let sprout_anchors_handle = self.db.cf_handle("sprout_anchors").unwrap(); + + self.db + .zs_items_in_range_unordered(&sprout_anchors_handle, ..) + } + /// Returns the Sapling note commitment tree of the finalized tip /// or the empty tree if the state is empty. pub fn sapling_note_commitment_tree(&self) -> Arc { diff --git a/zebra-test/Cargo.toml b/zebra-test/Cargo.toml index 4c712e3b..d64b6e59 100644 --- a/zebra-test/Cargo.toml +++ b/zebra-test/Cargo.toml @@ -21,7 +21,7 @@ lazy_static = "1.4.0" insta = "1.30.0" proptest = "1.2.0" once_cell = "1.18.0" -rand = { version = "0.8.5", package = "rand" } +rand = "0.8.5" regex = "1.8.4" tokio = { version = "1.28.2", features = ["full", "tracing", "test-util"] } diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 56c034e2..14e33dc1 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -172,7 +172,7 @@ dirs = "5.0.1" atty = "0.2.14" num-integer = "0.1.45" -rand = { version = "0.8.5", package = "rand" } +rand = "0.8.5" # prod feature sentry sentry = { version = "0.31.5", default-features = false, features = ["backtrace", "contexts", "reqwest", "rustls", "tracing"], optional = true }