From 3318eaaa22201229ddef1e40957648b1355bc48c Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 12 Dec 2023 17:45:12 +1000 Subject: [PATCH] test(scan): Add raw database format snapshots to the scanner (#8075) * Make some scanner storage methods more flexible * Move tests to a submodule and expose test functions and constants * Make scanner functions clearer and easier to use * Simplify state snapshot test code * Add raw data snapshot tests for the scanner * Add snapshots * Fix import path * Fix import conditional compilation * fix imports * fix imports 2 * Put read and write db exports together * Remove confusing IntoDisk/FromDisk impl * Fix an incorrect unused method that could panic * Delete a test that is no longer valid --------- Co-authored-by: Alfredo Garcia --- Cargo.lock | 3 + zebra-scan/Cargo.toml | 33 ++- zebra-scan/src/lib.rs | 4 +- zebra-scan/src/scan.rs | 6 +- zebra-scan/src/storage.rs | 13 +- zebra-scan/src/storage/db.rs | 3 + zebra-scan/src/storage/db/sapling.rs | 4 +- zebra-scan/src/storage/db/tests.rs | 3 + zebra-scan/src/storage/db/tests/snapshot.rs | 164 ++++++++++++++ .../tests/snapshots/column_family_names.snap | 8 + .../empty_column_families@empty.snap | 7 + .../empty_column_families@mainnet_0.snap | 5 + .../empty_column_families@mainnet_1.snap | 5 + .../empty_column_families@mainnet_2.snap | 5 + .../empty_column_families@mainnet_keys.snap | 5 + .../empty_column_families@testnet_0.snap | 5 + .../empty_column_families@testnet_1.snap | 5 + .../empty_column_families@testnet_2.snap | 5 + .../empty_column_families@testnet_keys.snap | 5 + .../sapling_tx_ids_raw_data@mainnet_0.snap | 18 ++ .../sapling_tx_ids_raw_data@mainnet_1.snap | 22 ++ .../sapling_tx_ids_raw_data@mainnet_2.snap | 26 +++ .../sapling_tx_ids_raw_data@mainnet_keys.snap | 14 ++ .../sapling_tx_ids_raw_data@testnet_0.snap | 18 ++ .../sapling_tx_ids_raw_data@testnet_1.snap | 22 ++ .../sapling_tx_ids_raw_data@testnet_2.snap | 26 +++ .../sapling_tx_ids_raw_data@testnet_keys.snap | 14 ++ zebra-scan/src/tests.rs | 204 ++---------------- zebra-scan/src/tests/vectors.rs | 198 +++++++++++++++++ zebra-state/src/lib.rs | 6 +- zebra-state/src/service/finalized_state.rs | 7 +- .../service/finalized_state/disk_format.rs | 5 +- .../finalized_state/disk_format/scan.rs | 42 ++-- .../disk_format/scan/tests/prop.rs | 7 - .../finalized_state/disk_format/tests.rs | 8 +- .../disk_format/tests/snapshot.rs | 25 +-- zebrad/Cargo.toml | 2 + zebrad/tests/acceptance.rs | 4 +- 38 files changed, 703 insertions(+), 253 deletions(-) create mode 100644 zebra-scan/src/storage/db/tests.rs create mode 100644 zebra-scan/src/storage/db/tests/snapshot.rs create mode 100644 zebra-scan/src/storage/db/tests/snapshots/column_family_names.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@empty.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_0.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_1.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_2.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_keys.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_0.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_1.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_2.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_keys.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_keys.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap create mode 100644 zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_keys.snap create mode 100644 zebra-scan/src/tests/vectors.rs diff --git a/Cargo.lock b/Cargo.lock index 6219729c..720fe491 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5885,8 +5885,11 @@ dependencies = [ "ff", "group", "indexmap 2.1.0", + "insta", "itertools 0.12.0", "jubjub", + "proptest", + "proptest-derive", "rand 0.8.5", "semver 1.0.20", "serde", diff --git a/zebra-scan/Cargo.toml b/zebra-scan/Cargo.toml index f4884e2f..f9b10750 100644 --- a/zebra-scan/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -18,6 +18,21 @@ categories = ["cryptography::cryptocurrencies"] # Production features that activate extra dependencies, or extra features in dependencies +# Test features + +proptest-impl = [ + "proptest", + "proptest-derive", + "zebra-state/proptest-impl", + "zebra-chain/proptest-impl", + "bls12_381", + "ff", + "group", + "jubjub", + "rand", + "zcash_note_encryption", +] + [dependencies] color-eyre = "0.6.2" @@ -37,15 +52,29 @@ zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = [ chrono = { version = "0.4.31", default-features = false, features = ["clock", "std", "serde"] } +# test feature proptest-impl +proptest = { version = "1.4.0", optional = true } +proptest-derive = { version = "0.4.0", optional = true } + +bls12_381 = { version = "0.8.0", optional = true } +ff = { version = "0.13.0", optional = true } +group = { version = "0.13.0", optional = true } +jubjub = { version = "0.10.0", optional = true } +rand = { version = "0.8.5", optional = true } +zcash_note_encryption = { version = "0.4.0", optional = true } + [dev-dependencies] +insta = { version = "1.33.0", features = ["ron", "redactions"] } +tokio = { version = "1.34.0", features = ["test-util"] } + +proptest = "1.4.0" +proptest-derive = "0.4.0" bls12_381 = "0.8.0" ff = "0.13.0" group = "0.13.0" jubjub = "0.10.0" rand = "0.8.5" -tokio = { version = "1.34.0", features = ["test-util"] } - zcash_note_encryption = "0.4.0" zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = ["proptest-impl"] } diff --git a/zebra-scan/src/lib.rs b/zebra-scan/src/lib.rs index b31fa99d..eee89247 100644 --- a/zebra-scan/src/lib.rs +++ b/zebra-scan/src/lib.rs @@ -9,8 +9,8 @@ pub mod init; pub mod scan; pub mod storage; -#[cfg(test)] -mod tests; +#[cfg(any(test, feature = "proptest-impl"))] +pub mod tests; pub use config::Config; pub use init::{init, spawn_init}; diff --git a/zebra-scan/src/scan.rs b/zebra-scan/src/scan.rs index 0db04523..f9d5d0f4 100644 --- a/zebra-scan/src/scan.rs +++ b/zebra-scan/src/scan.rs @@ -211,8 +211,8 @@ pub async fn scan_height_and_store_results( let dfvk_res = scanned_block_to_db_result(dfvk_res); let ivk_res = scanned_block_to_db_result(ivk_res); - storage.add_sapling_results(sapling_key.clone(), height, dfvk_res); - storage.add_sapling_results(sapling_key, height, ivk_res); + storage.add_sapling_results(&sapling_key, height, dfvk_res); + storage.add_sapling_results(&sapling_key, height, ivk_res); Ok::<_, Report>(()) }) @@ -398,7 +398,7 @@ fn scanned_block_to_db_result( .map(|tx| { ( TransactionIndex::from_usize(tx.index), - SaplingScannedResult::from(tx.txid.as_ref()), + SaplingScannedResult::from_bytes_in_display_order(*tx.txid.as_ref()), ) }) .collect() diff --git a/zebra-scan/src/storage.rs b/zebra-scan/src/storage.rs index c4324ced..10f1b1d3 100644 --- a/zebra-scan/src/storage.rs +++ b/zebra-scan/src/storage.rs @@ -71,7 +71,13 @@ impl Storage { /// /// This method can block while writing database files, so it must be inside spawn_blocking() /// in async code. - pub fn add_sapling_key(&mut self, sapling_key: &SaplingScanningKey, birthday: Option) { + pub fn add_sapling_key( + &mut self, + sapling_key: &SaplingScanningKey, + birthday: impl Into>, + ) { + let birthday = birthday.into(); + // It's ok to write some keys and not others during shutdown, so each key can get its own // batch. (They will be re-written on startup anyway.) let mut batch = ScannerWriteBatch::default(); @@ -93,7 +99,8 @@ impl Storage { self.sapling_keys_and_birthday_heights() } - /// Add the sapling results for `height` to the storage. + /// Add the sapling results for `height` to the storage. The results can be any map of + /// [`TransactionIndex`] to [`SaplingScannedResult`]. /// /// # Performance / Hangs /// @@ -101,7 +108,7 @@ impl Storage { /// in async code. pub fn add_sapling_results( &mut self, - sapling_key: SaplingScanningKey, + sapling_key: &SaplingScanningKey, height: Height, sapling_results: BTreeMap, ) { diff --git a/zebra-scan/src/storage/db.rs b/zebra-scan/src/storage/db.rs index 00262b01..12a3a404 100644 --- a/zebra-scan/src/storage/db.rs +++ b/zebra-scan/src/storage/db.rs @@ -19,6 +19,9 @@ pub use zebra_state::{ pub mod sapling; +#[cfg(test)] +mod tests; + /// The directory name used to distinguish the scanner database from Zebra's other databases or /// flat files. /// diff --git a/zebra-scan/src/storage/db/sapling.rs b/zebra-scan/src/storage/db/sapling.rs index b47d344a..db699207 100644 --- a/zebra-scan/src/storage/db/sapling.rs +++ b/zebra-scan/src/storage/db/sapling.rs @@ -47,13 +47,15 @@ impl Storage { // Reading Sapling database entries /// Returns the result for a specific database index (key, block height, transaction index). + /// Returns `None` if the result is missing or an empty marker for a birthday or progress + /// height. // // TODO: add tests for this method pub fn sapling_result_for_index( &self, index: &SaplingScannedDatabaseIndex, ) -> Option { - self.db.zs_get(&self.sapling_tx_ids_cf(), &index) + self.db.zs_get(&self.sapling_tx_ids_cf(), &index).flatten() } /// Returns the results for a specific key and block height. diff --git a/zebra-scan/src/storage/db/tests.rs b/zebra-scan/src/storage/db/tests.rs new file mode 100644 index 00000000..ca39f410 --- /dev/null +++ b/zebra-scan/src/storage/db/tests.rs @@ -0,0 +1,3 @@ +//! General scanner database tests. + +mod snapshot; diff --git a/zebra-scan/src/storage/db/tests/snapshot.rs b/zebra-scan/src/storage/db/tests/snapshot.rs new file mode 100644 index 00000000..0eaef193 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshot.rs @@ -0,0 +1,164 @@ +//! Raw data snapshot tests for the scanner database format. +//! +//! These tests check: +//! - the name of each column family +//! - the number of key-value entries +//! - the bytes in each key and value +//! +//! These tests currently use fixed test vectors. +//! +//! # Fixing Test Failures +//! +//! If this test fails, run: +//! ```sh +//! cd zebra-scan +//! cargo insta test --review --features shielded-scan +//! ``` +//! to update the test snapshots, then commit the `test_*.snap` files using git. +//! +//! # Snapshot Format +//! +//! These snapshots use [RON (Rusty Object Notation)](https://github.com/ron-rs/ron#readme), +//! a text format similar to Rust syntax. Raw byte data is encoded in hexadecimal. +//! +//! Due to `serde` limitations, some object types can't be represented exactly, +//! so RON uses the closest equivalent structure. + +use std::{collections::BTreeMap, sync::Arc}; + +use zebra_chain::{ + block::{Block, Height}, + parameters::Network::{self, *}, + serialization::ZcashDeserializeInto, +}; +use zebra_state::{RawBytes, ReadDisk, TransactionIndex, KV}; + +use crate::{ + storage::{db::ScannerDb, Storage}, + tests::{FAKE_SAPLING_VIEWING_KEY, ZECPAGES_SAPLING_VIEWING_KEY}, + Config, +}; + +/// Snapshot test for RocksDB column families, and their key-value data. +/// +/// These snapshots contain the `default` column family, but it is not used by Zebra. +#[test] +fn test_raw_rocksdb_column_families() { + let _init_guard = zebra_test::init(); + + test_raw_rocksdb_column_families_with_network(Mainnet); + test_raw_rocksdb_column_families_with_network(Testnet); +} + +/// Snapshot raw column families for `network`. +/// +/// See [`test_raw_rocksdb_column_families`]. +fn test_raw_rocksdb_column_families_with_network(network: Network) { + let mut net_suffix = network.to_string(); + net_suffix.make_ascii_lowercase(); + + let mut storage = Storage::new(&Config::ephemeral(), network); + + // Snapshot the column family names + let mut cf_names = storage.db.list_cf().expect("empty database is valid"); + + // The order that RocksDB returns column families is irrelevant, + // because we always access them by name. + cf_names.sort(); + + // Assert that column family names are the same, regardless of the network. + // Later, we check they are also the same regardless of the block height. + insta::assert_ron_snapshot!("column_family_names", cf_names); + + // Assert that empty databases are the same, regardless of the network. + let mut settings = insta::Settings::clone_current(); + + settings.set_snapshot_suffix("empty"); + settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names)); + + // Snapshot a birthday that is automatically set to activation height + storage.add_sapling_key(&ZECPAGES_SAPLING_VIEWING_KEY.to_string(), None); + // Snapshot a birthday above activation height + storage.add_sapling_key(&FAKE_SAPLING_VIEWING_KEY.to_string(), Height(1_000_000)); + + settings.set_snapshot_suffix(format!("{net_suffix}_keys")); + settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names)); + + // Snapshot raw database data for: + // - mainnet and testnet + // - genesis, block 1, and block 2 + let blocks = match network { + Mainnet => &*zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS, + Testnet => &*zebra_test::vectors::CONTINUOUS_TESTNET_BLOCKS, + }; + + // We limit the number of blocks, because the serialized data is a few kilobytes per block. + for height in 0..=2 { + let block: Arc = blocks + .get(&height) + .expect("block height has test data") + .zcash_deserialize_into() + .expect("test data deserializes"); + + // Fake results from the first few blocks + storage.add_sapling_results( + &ZECPAGES_SAPLING_VIEWING_KEY.to_string(), + Height(height), + block + .transactions + .iter() + .enumerate() + .map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into())) + .collect(), + ); + + let mut settings = insta::Settings::clone_current(); + settings.set_snapshot_suffix(format!("{net_suffix}_{height}")); + + settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names)); + } +} + +/// Snapshot the data in each column family, using `cargo insta` and RON serialization. +fn snapshot_raw_rocksdb_column_family_data(db: &ScannerDb, original_cf_names: &[String]) { + let mut new_cf_names = db.list_cf().expect("empty database is valid"); + new_cf_names.sort(); + + // Assert that column family names are the same, regardless of the network or block height. + assert_eq!( + original_cf_names, new_cf_names, + "unexpected extra column families", + ); + + let mut empty_column_families = Vec::new(); + + // Now run the data snapshots + for cf_name in original_cf_names { + let cf_handle = db + .cf_handle(cf_name) + .expect("RocksDB API provides correct names"); + + // Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests. + let cf_items: BTreeMap = db.zs_items_in_range_ordered(&cf_handle, ..); + + // The default raw data serialization is very verbose, so we hex-encode the bytes. + let cf_data: Vec = cf_items + .iter() + .map(|(key, value)| KV::new(key.raw_bytes(), value.raw_bytes())) + .collect(); + + if cf_name == "default" { + assert_eq!(cf_data.len(), 0, "default column family is never used"); + } else if cf_data.is_empty() { + // distinguish column family names from empty column families + empty_column_families.push(format!("{cf_name}: no entries")); + } else { + // The note commitment tree snapshots will change if the trees do not have cached roots. + // But we expect them to always have cached roots, + // because those roots are used to populate the anchor column families. + insta::assert_ron_snapshot!(format!("{cf_name}_raw_data"), cf_data); + } + } + + insta::assert_ron_snapshot!("empty_column_families", empty_column_families); +} diff --git a/zebra-scan/src/storage/db/tests/snapshots/column_family_names.snap b/zebra-scan/src/storage/db/tests/snapshots/column_family_names.snap new file mode 100644 index 00000000..dc65b126 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/column_family_names.snap @@ -0,0 +1,8 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_names +--- +[ + "default", + "sapling_tx_ids", +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@empty.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@empty.snap new file mode 100644 index 00000000..df6ef1d5 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@empty.snap @@ -0,0 +1,7 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[ + "sapling_tx_ids: no entries", +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_0.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_0.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_1.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_1.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_2.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_2.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_keys.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_keys.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_keys.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_0.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_0.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_1.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_1.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_2.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_2.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_keys.snap b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_keys.snap new file mode 100644 index 00000000..15b9ea62 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_keys.snap @@ -0,0 +1,5 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: empty_column_families +--- +[] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap new file mode 100644 index 00000000..7e4978db --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap @@ -0,0 +1,18 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000", + v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap new file mode 100644 index 00000000..710d65d2 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap @@ -0,0 +1,22 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000", + v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000", + v: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap new file mode 100644 index 00000000..fae6f22c --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap @@ -0,0 +1,26 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000", + v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000", + v: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000020000", + v: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_keys.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_keys.snap new file mode 100644 index 00000000..34ac710b --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_keys.snap @@ -0,0 +1,14 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap new file mode 100644 index 00000000..53089e84 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap @@ -0,0 +1,18 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000", + v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap new file mode 100644 index 00000000..4384d29f --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap @@ -0,0 +1,22 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000", + v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000", + v: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap new file mode 100644 index 00000000..a2e7dfce --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap @@ -0,0 +1,26 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000", + v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000", + v: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000020000", + v: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5", + ), + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_keys.snap b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_keys.snap new file mode 100644 index 00000000..19340f88 --- /dev/null +++ b/zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_keys.snap @@ -0,0 +1,14 @@ +--- +source: zebra-scan/src/storage/db/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000", + v: "", + ), + KV( + k: "7a78766965777366616b650f423f0000", + v: "", + ), +] diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index 99b8c87d..55eef898 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -12,17 +12,14 @@ use ff::{Field, PrimeField}; use group::GroupEncoding; use rand::{rngs::OsRng, thread_rng, RngCore}; -use zcash_client_backend::{ - encoding::decode_extended_full_viewing_key, - proto::compact_formats::{ - ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, - }, +use zcash_client_backend::proto::compact_formats::{ + ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }; use zcash_note_encryption::Domain; use zcash_primitives::{ block::BlockHash, consensus::BlockHeight, - constants::{mainnet::HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, SPENDING_KEY_GENERATOR}, + constants::SPENDING_KEY_GENERATOR, memo::MemoBytes, sapling::{ note_encryption::{sapling_note_encryption, SaplingDomain}, @@ -30,200 +27,29 @@ use zcash_primitives::{ value::NoteValue, Note, Nullifier, }, - zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey}, + zip32::DiversifiableFullViewingKey, }; use zebra_chain::{ amount::{Amount, NegativeAllowed}, block::{self, merkle, Block, Header, Height}, - chain_tip::ChainTip, fmt::HexDebug, - parameters::Network, primitives::{redjubjub, Groth16Proof}, sapling::{self, PerSpendAnchor, Spend, TransferData}, - serialization::{AtLeastOne, ZcashDeserializeInto}, + serialization::AtLeastOne, transaction::{LockTime, Transaction}, transparent::{CoinbaseData, Input}, work::{difficulty::CompactDifficulty, equihash::Solution}, }; -use zebra_state::{SaplingScannedResult, TransactionIndex}; -use crate::{ - config::Config, - scan::{block_to_compact, scan_block}, -}; +#[cfg(test)] +mod vectors; -/// This test: -/// - Creates a viewing key and a fake block containing a Sapling output decryptable by the key. -/// - Scans the block. -/// - Checks that the result contains the txid of the tx containing the Sapling output. -#[tokio::test] -async fn scanning_from_fake_generated_blocks() -> Result<()> { - let extsk = ExtendedSpendingKey::master(&[]); - let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); - let nf = Nullifier([7; 32]); +/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) +pub const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; - let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0)); - - assert_eq!(block.transactions.len(), 4); - - let res = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap(); - - // The response should have one transaction relevant to the key we provided. - assert_eq!(res.transactions().len(), 1); - - // Check that the original block contains the txid in the scanning result. - assert!(block - .transactions - .iter() - .map(|tx| tx.hash().bytes_in_display_order()) - .any(|txid| &txid == res.transactions()[0].txid.as_ref())); - - // Check that the txid in the scanning result matches the third tx in the original block. - assert_eq!( - res.transactions()[0].txid.as_ref(), - &block.transactions[2].hash().bytes_in_display_order() - ); - - // The block hash of the response should be the same as the one provided. - assert_eq!(res.block_hash().0, block.hash().0); - - Ok(()) -} - -/// Scan a populated state for the ZECpages viewing key. -/// This test is very similar to `scanning_from_populated_zebra_state` but with the ZECpages key. -/// There are no zechub transactions in the test data so we should get empty related transactions. -#[tokio::test] -async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { - /// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) - const ZECPAGES_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; - - // Parse the key from ZECpages - let efvk = decode_extended_full_viewing_key( - HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, - ZECPAGES_VIEWING_KEY, - ) - .unwrap(); - - // Build a vector of viewing keys `vks` to scan for. - let fvk = efvk.fvk; - let ivk = fvk.vk.ivk(); - let ivks = vec![ivk]; - - let network = Network::Mainnet; - - // Create a continuous chain of mainnet blocks from genesis - let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS - .iter() - .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) - .collect(); - - // Create a populated state service. - let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) = - zebra_state::populated_state(blocks.clone(), network).await; - - let db = read_only_state_service.db(); - - // use the tip as starting height - let mut height = latest_chain_tip.best_tip_height().unwrap(); - - let mut transactions_found = 0; - let mut transactions_scanned = 0; - let mut blocks_scanned = 0; - while let Some(block) = db.block(height.into()) { - // We use a dummy size of the Sapling note commitment tree. We can't set the size to zero - // because the underlying scanning function would return - // `zcash_client_backeng::scanning::ScanError::TreeSizeUnknown`. - let sapling_commitment_tree_size = 1; - - let orchard_commitment_tree_size = 0; - - let chain_metadata = ChainMetadata { - sapling_commitment_tree_size, - orchard_commitment_tree_size, - }; - - let compact_block = block_to_compact(&block, chain_metadata); - - let res = scan_block(network, &block, sapling_commitment_tree_size, &ivks) - .expect("scanning block for the ZECpages viewing key should work"); - - transactions_found += res.transactions().len(); - transactions_scanned += compact_block.vtx.len(); - blocks_scanned += 1; - - // scan backwards - if height.is_min() { - break; - } - height = height.previous()?; - } - - // make sure all blocks and transactions were scanned - assert_eq!(blocks_scanned, 11); - assert_eq!(transactions_scanned, 11); - - // no relevant transactions should be found - assert_eq!(transactions_found, 0); - - Ok(()) -} - -/// Creates a viewing key and a fake block containing a Sapling output decryptable by the key, scans -/// the block using the key, and adds the results to the database. -/// -/// The purpose of this test is to check if our database and our scanning code are compatible. -#[test] -#[allow(deprecated)] -fn scanning_fake_blocks_store_key_and_results() -> Result<()> { - // Generate a key - let extsk = ExtendedSpendingKey::master(&[]); - // TODO: find out how to do it with `to_diversifiable_full_viewing_key` as `to_extended_full_viewing_key` is deprecated. - let extfvk = extsk.to_extended_full_viewing_key(); - let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); - let key_to_be_stored = - zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk); - - // Create a database - let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet); - - // Insert the generated key to the database - s.add_sapling_key(&key_to_be_stored, None); - - // Check key was added - assert_eq!(s.sapling_keys().len(), 1); - assert_eq!( - s.sapling_keys().get(&key_to_be_stored), - Some(&s.min_sapling_birthday_height()) - ); - - let nf = Nullifier([7; 32]); - - let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0)); - - let result = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap(); - - // The response should have one transaction relevant to the key we provided. - assert_eq!(result.transactions().len(), 1); - - let result = SaplingScannedResult::from(result.transactions()[0].txid.as_ref()); - - // Add result to database - s.add_sapling_results( - key_to_be_stored.clone(), - Height(1), - [(TransactionIndex::from_usize(0), result)].into(), - ); - - // Check the result was added - assert_eq!( - s.sapling_results(&key_to_be_stored).get(&Height(1)), - Some(&vec![result]) - ); - - Ok(()) -} +/// A fake viewing key in an incorrect format. +pub const FAKE_SAPLING_VIEWING_KEY: &str = "zxviewsfake"; /// Generates a fake block containing a Sapling output decryptable by `dfvk`. /// @@ -232,7 +58,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> { /// 2. a V4 tx containing a random Sapling output, /// 3. a V4 tx containing a Sapling output decryptable by `dfvk`, /// 4. depending on the value of `tx_after`, another V4 tx containing a random Sapling output. -fn fake_block( +pub fn fake_block( height: BlockHeight, nf: Nullifier, dfvk: &DiversifiableFullViewingKey, @@ -305,7 +131,7 @@ fn fake_block( // be a number for easier conversion: // https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L635 // We need to copy because this is a test private function upstream. -fn fake_compact_block( +pub fn fake_compact_block( height: BlockHeight, prev_hash: BlockHash, nf: Nullifier, @@ -394,7 +220,7 @@ fn fake_compact_block( // This is an exact copy of `zcash_client_backend::scanning::random_compact_tx`: // https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L597 // We need to copy because this is a test private function upstream. -fn random_compact_tx(mut rng: impl RngCore) -> CompactTx { +pub fn random_compact_tx(mut rng: impl RngCore) -> CompactTx { let fake_nf = { let mut nf = vec![0; 32]; rng.fill_bytes(&mut nf); @@ -427,7 +253,7 @@ fn random_compact_tx(mut rng: impl RngCore) -> CompactTx { } /// Converts [`CompactTx`] to [`Transaction::V4`]. -fn compact_to_v4(tx: &CompactTx) -> Result { +pub fn compact_to_v4(tx: &CompactTx) -> Result { let sk = redjubjub::SigningKey::::new(thread_rng()); let vk = redjubjub::VerificationKey::from(&sk); let dummy_rk = sapling::keys::ValidatingKey::try_from(vk) diff --git a/zebra-scan/src/tests/vectors.rs b/zebra-scan/src/tests/vectors.rs new file mode 100644 index 00000000..bcdca24c --- /dev/null +++ b/zebra-scan/src/tests/vectors.rs @@ -0,0 +1,198 @@ +//! Fixed integration test vectors for the scanner. + +use std::sync::Arc; + +use color_eyre::Result; + +use zcash_client_backend::{ + encoding::decode_extended_full_viewing_key, proto::compact_formats::ChainMetadata, +}; +use zcash_primitives::{ + constants::mainnet::HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, + sapling::Nullifier, + zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey}, +}; + +use zebra_chain::{ + block::{Block, Height}, + chain_tip::ChainTip, + parameters::Network, + serialization::ZcashDeserializeInto, +}; +use zebra_state::{SaplingScannedResult, TransactionIndex}; + +use crate::{ + config::Config, + scan::{block_to_compact, scan_block}, + tests::{fake_block, ZECPAGES_SAPLING_VIEWING_KEY}, +}; + +/// This test: +/// - Creates a viewing key and a fake block containing a Sapling output decryptable by the key. +/// - Scans the block. +/// - Checks that the result contains the txid of the tx containing the Sapling output. +#[tokio::test] +async fn scanning_from_fake_generated_blocks() -> Result<()> { + let extsk = ExtendedSpendingKey::master(&[]); + let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); + let nf = Nullifier([7; 32]); + + let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0)); + + assert_eq!(block.transactions.len(), 4); + + let res = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap(); + + // The response should have one transaction relevant to the key we provided. + assert_eq!(res.transactions().len(), 1); + + // Check that the original block contains the txid in the scanning result. + assert!(block + .transactions + .iter() + .map(|tx| tx.hash().bytes_in_display_order()) + .any(|txid| &txid == res.transactions()[0].txid.as_ref())); + + // Check that the txid in the scanning result matches the third tx in the original block. + assert_eq!( + res.transactions()[0].txid.as_ref(), + &block.transactions[2].hash().bytes_in_display_order() + ); + + // The block hash of the response should be the same as the one provided. + assert_eq!(res.block_hash().0, block.hash().0); + + Ok(()) +} + +/// Scan a populated state for the ZECpages viewing key. +/// This test is very similar to `scanning_from_populated_zebra_state` but with the ZECpages key. +/// There are no zechub transactions in the test data so we should get empty related transactions. +#[tokio::test] +async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { + // Parse the key from ZECpages + let efvk = decode_extended_full_viewing_key( + HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, + ZECPAGES_SAPLING_VIEWING_KEY, + ) + .unwrap(); + + // Build a vector of viewing keys `vks` to scan for. + let fvk = efvk.fvk; + let ivk = fvk.vk.ivk(); + let ivks = vec![ivk]; + + let network = Network::Mainnet; + + // Create a continuous chain of mainnet blocks from genesis + let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .iter() + .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) + .collect(); + + // Create a populated state service. + let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) = + zebra_state::populated_state(blocks.clone(), network).await; + + let db = read_only_state_service.db(); + + // use the tip as starting height + let mut height = latest_chain_tip.best_tip_height().unwrap(); + + let mut transactions_found = 0; + let mut transactions_scanned = 0; + let mut blocks_scanned = 0; + while let Some(block) = db.block(height.into()) { + // We use a dummy size of the Sapling note commitment tree. We can't set the size to zero + // because the underlying scanning function would return + // `zcash_client_backeng::scanning::ScanError::TreeSizeUnknown`. + let sapling_commitment_tree_size = 1; + + let orchard_commitment_tree_size = 0; + + let chain_metadata = ChainMetadata { + sapling_commitment_tree_size, + orchard_commitment_tree_size, + }; + + let compact_block = block_to_compact(&block, chain_metadata); + + let res = scan_block(network, &block, sapling_commitment_tree_size, &ivks) + .expect("scanning block for the ZECpages viewing key should work"); + + transactions_found += res.transactions().len(); + transactions_scanned += compact_block.vtx.len(); + blocks_scanned += 1; + + // scan backwards + if height.is_min() { + break; + } + height = height.previous()?; + } + + // make sure all blocks and transactions were scanned + assert_eq!(blocks_scanned, 11); + assert_eq!(transactions_scanned, 11); + + // no relevant transactions should be found + assert_eq!(transactions_found, 0); + + Ok(()) +} + +/// Creates a viewing key and a fake block containing a Sapling output decryptable by the key, scans +/// the block using the key, and adds the results to the database. +/// +/// The purpose of this test is to check if our database and our scanning code are compatible. +#[test] +#[allow(deprecated)] +fn scanning_fake_blocks_store_key_and_results() -> Result<()> { + // Generate a key + let extsk = ExtendedSpendingKey::master(&[]); + // TODO: find out how to do it with `to_diversifiable_full_viewing_key` as `to_extended_full_viewing_key` is deprecated. + let extfvk = extsk.to_extended_full_viewing_key(); + let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); + let key_to_be_stored = + zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk); + + // Create a database + let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet); + + // Insert the generated key to the database + s.add_sapling_key(&key_to_be_stored, None); + + // Check key was added + assert_eq!(s.sapling_keys().len(), 1); + assert_eq!( + s.sapling_keys().get(&key_to_be_stored), + Some(&s.min_sapling_birthday_height()) + ); + + let nf = Nullifier([7; 32]); + + let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0)); + + let result = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap(); + + // The response should have one transaction relevant to the key we provided. + assert_eq!(result.transactions().len(), 1); + + let result = + SaplingScannedResult::from_bytes_in_display_order(*result.transactions()[0].txid.as_ref()); + + // Add result to database + s.add_sapling_results( + &key_to_be_stored, + Height(1), + [(TransactionIndex::from_usize(0), result)].into(), + ); + + // Check the result was added + assert_eq!( + s.sapling_results(&key_to_be_stored).get(&Height(1)), + Some(&vec![result]) + ); + + Ok(()) +} diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 09c31577..6dbffd9a 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -63,12 +63,12 @@ pub use service::{ pub use rocksdb::AsColumnFamilyRef; #[cfg(feature = "shielded-scan")] pub use service::finalized_state::{ - FromDisk, IntoDisk, ReadDisk, SaplingScannedDatabaseEntry, SaplingScannedDatabaseIndex, + FromDisk, IntoDisk, SaplingScannedDatabaseEntry, SaplingScannedDatabaseIndex, SaplingScannedResult, SaplingScanningKey, ZebraDb, }; #[cfg(any(test, feature = "proptest-impl", feature = "shielded-scan"))] -pub use service::finalized_state::{DiskWriteBatch, WriteDisk}; +pub use service::finalized_state::{DiskWriteBatch, ReadDisk, WriteDisk}; #[cfg(feature = "getblocktemplate-rpcs")] pub use response::GetBlockTemplateChainInfo; @@ -77,7 +77,7 @@ pub use response::GetBlockTemplateChainInfo; pub use service::{ arbitrary::{populated_state, CHAIN_TIP_UPDATE_WAIT_LIMIT}, chain_tip::{ChainTipBlock, ChainTipSender}, - finalized_state::MAX_ON_DISK_HEIGHT, + finalized_state::{RawBytes, KV, MAX_ON_DISK_HEIGHT}, init_test, init_test_services, ReadStateService, }; diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index 2e923fe7..151ed012 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -42,8 +42,8 @@ mod tests; pub use disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}; #[allow(unused_imports)] pub use disk_format::{ - FromDisk, IntoDisk, OutputIndex, OutputLocation, TransactionIndex, TransactionLocation, - MAX_ON_DISK_HEIGHT, + FromDisk, IntoDisk, OutputIndex, OutputLocation, RawBytes, TransactionIndex, + TransactionLocation, MAX_ON_DISK_HEIGHT, }; pub use zebra_db::ZebraDb; @@ -53,6 +53,9 @@ pub use disk_format::{ SaplingScanningKey, }; +#[cfg(any(test, feature = "proptest-impl"))] +pub use disk_format::KV; + /// The column families supported by the running `zebra-state` database code. /// /// Existing column families that aren't listed here are preserved when the database is opened. diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs index 11ef2aff..0ce04431 100644 --- a/zebra-state/src/service/finalized_state/disk_format.rs +++ b/zebra-state/src/service/finalized_state/disk_format.rs @@ -16,7 +16,7 @@ pub mod upgrade; #[cfg(feature = "shielded-scan")] pub mod scan; -#[cfg(test)] +#[cfg(any(test, feature = "proptest-impl"))] mod tests; pub use block::{TransactionIndex, TransactionLocation, MAX_ON_DISK_HEIGHT}; @@ -28,6 +28,9 @@ pub use scan::{ SaplingScanningKey, }; +#[cfg(any(test, feature = "proptest-impl"))] +pub use tests::KV; + /// Helper type for writing types to disk as raw bytes. /// Also used to convert key types to raw bytes for disk lookups. pub trait IntoDisk { diff --git a/zebra-state/src/service/finalized_state/disk_format/scan.rs b/zebra-state/src/service/finalized_state/disk_format/scan.rs index 4d4b3b81..9c14800c 100644 --- a/zebra-state/src/service/finalized_state/disk_format/scan.rs +++ b/zebra-state/src/service/finalized_state/disk_format/scan.rs @@ -37,9 +37,21 @@ impl From for transaction::Hash { } } -impl From<&[u8; 32]> for SaplingScannedResult { - fn from(bytes: &[u8; 32]) -> Self { - Self(*bytes) +impl From for SaplingScannedResult { + fn from(hash: transaction::Hash) -> Self { + SaplingScannedResult(hash.bytes_in_display_order()) + } +} + +impl SaplingScannedResult { + /// Creates a `SaplingScannedResult` from bytes in display order. + pub fn from_bytes_in_display_order(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the inner bytes in display order. + pub fn bytes_in_display_order(&self) -> [u8; 32] { + self.0 } } @@ -162,19 +174,8 @@ impl FromDisk for SaplingScannedDatabaseIndex { } } -impl IntoDisk for SaplingScannedResult { - type Bytes = [u8; 32]; - - fn as_bytes(&self) -> Self::Bytes { - self.0 - } -} - -impl FromDisk for SaplingScannedResult { - fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { - SaplingScannedResult(bytes.as_ref().try_into().unwrap()) - } -} +// We can't implement IntoDisk or FromDisk for SaplingScannedResult, +// because the format is actually Option. impl IntoDisk for Option { type Bytes = Vec; @@ -183,7 +184,7 @@ impl IntoDisk for Option { let mut bytes = Vec::new(); if let Some(result) = self.as_ref() { - bytes.extend(result.as_bytes()); + bytes.extend(result.bytes_in_display_order()); } bytes @@ -191,13 +192,18 @@ impl IntoDisk for Option { } impl FromDisk for Option { + #[allow(clippy::unwrap_in_result)] fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { let bytes = bytes.as_ref(); if bytes.is_empty() { None } else { - Some(SaplingScannedResult::from_bytes(bytes)) + Some(SaplingScannedResult::from_bytes_in_display_order( + bytes + .try_into() + .expect("unexpected incorrect SaplingScannedResult data length"), + )) } } } diff --git a/zebra-state/src/service/finalized_state/disk_format/scan/tests/prop.rs b/zebra-state/src/service/finalized_state/disk_format/scan/tests/prop.rs index fa964370..c770daec 100644 --- a/zebra-state/src/service/finalized_state/disk_format/scan/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/disk_format/scan/tests/prop.rs @@ -29,13 +29,6 @@ fn roundtrip_sapling_db_index() { ); } -#[test] -fn roundtrip_sapling_result() { - let _init_guard = zebra_test::init(); - - proptest!(|(val in any::())| assert_value_properties(val)); -} - #[test] fn roundtrip_option_sapling_result() { let _init_guard = zebra_test::init(); diff --git a/zebra-state/src/service/finalized_state/disk_format/tests.rs b/zebra-state/src/service/finalized_state/disk_format/tests.rs index 4680764c..f959800b 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests.rs @@ -2,12 +2,14 @@ use serde::{Deserialize, Serialize}; +#[cfg(test)] mod prop; +#[cfg(test)] mod snapshot; /// A formatting struct for raw key-value data #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] -struct KV { +pub struct KV { /// The raw key bytes, as a hexadecimal-encoded string. k: String, @@ -17,7 +19,7 @@ struct KV { impl KV { /// Create a new `KV` from raw key-value data. - fn new(key: K, value: V) -> KV + pub fn new(key: K, value: V) -> KV where K: AsRef<[u8]>, V: AsRef<[u8]>, @@ -26,7 +28,7 @@ impl KV { } /// Create a new `KV` from hex-encoded key-value data. - fn new_hex(key: String, value: String) -> KV { + pub fn new_hex(key: String, value: String) -> KV { KV { k: key, v: value } } } diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs index 67b4f2eb..5866b08d 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs @@ -27,7 +27,7 @@ //! //! Test shielded data, and data activated in Overwinter and later network upgrades. -use std::sync::Arc; +use std::{collections::BTreeMap, sync::Arc}; use zebra_chain::{ block::Block, @@ -37,11 +37,11 @@ use zebra_chain::{ use crate::{ service::finalized_state::{ - disk_db::{DiskDb, DB}, - disk_format::tests::KV, + disk_db::DiskDb, + disk_format::{tests::KV, RawBytes}, FinalizedState, }, - Config, + Config, ReadDisk, }; /// Snapshot test for RocksDB column families, and their key-value data. @@ -133,13 +133,12 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str .expect("RocksDB API provides correct names"); // Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests. - let mut cf_iter = db.full_iterator_cf(&cf_handle, rocksdb::IteratorMode::Start); + let cf_items: BTreeMap = db.zs_items_in_range_ordered(&cf_handle, ..); // The default raw data serialization is very verbose, so we hex-encode the bytes. - let cf_data: Vec = cf_iter - .by_ref() - .map(|result| result.expect("unexpected database error")) - .map(|(key, value)| KV::new(key, value)) + let cf_data: Vec = cf_items + .iter() + .map(|(key, value)| KV::new(key.raw_bytes(), value.raw_bytes())) .collect(); if cf_name == "default" { @@ -153,14 +152,6 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str // because those roots are used to populate the anchor column families. insta::assert_ron_snapshot!(format!("{cf_name}_raw_data"), cf_data); } - - let raw_cf_iter: rocksdb::DBRawIteratorWithThreadMode = cf_iter.into(); - - assert_eq!( - raw_cf_iter.status(), - Ok(()), - "unexpected column family iterator error", - ); } insta::assert_ron_snapshot!("empty_column_families", empty_column_families); diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 2e043552..c71bea1e 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -118,6 +118,7 @@ proptest-impl = [ "zebra-state/proptest-impl", "zebra-network/proptest-impl", "zebra-chain/proptest-impl", + "zebra-scan?/proptest-impl", ] # Build the zebra-checkpoints utility for checkpoint generation tests @@ -271,6 +272,7 @@ zebra-chain = { path = "../zebra-chain", features = ["proptest-impl"] } zebra-consensus = { path = "../zebra-consensus", features = ["proptest-impl"] } zebra-network = { path = "../zebra-network", features = ["proptest-impl"] } zebra-state = { path = "../zebra-state", features = ["proptest-impl"] } +zebra-scan = { path = "../zebra-scan", features = ["proptest-impl"] } zebra-node-services = { path = "../zebra-node-services", features = ["rpc-client"] } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index c2cb2ec9..94a8066a 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -2821,13 +2821,13 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> { fn scan_task_starts() -> Result<()> { use indexmap::IndexMap; - const ZECPAGES_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; + use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY; let _init_guard = zebra_test::init(); let mut config = default_test_config(Mainnet)?; let mut keys = IndexMap::new(); - keys.insert(ZECPAGES_VIEWING_KEY.to_string(), 1); + keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1); config.shielded_scan.sapling_keys_to_scan = keys; let testdir = testdir()?.with_config(&mut config)?;