test(scan): Add raw database format snapshots to the scanner (#8075)

* Make some scanner storage methods more flexible

* Move tests to a submodule and expose test functions and constants

* Make scanner functions clearer and easier to use

* Simplify state snapshot test code

* Add raw data snapshot tests for the scanner

* Add snapshots

* Fix import path

* Fix import conditional compilation

* fix imports

* fix imports 2

* Put read and write db exports together

* Remove confusing IntoDisk/FromDisk impl

* Fix an incorrect unused method that could panic

* Delete a test that is no longer valid

---------

Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>

Cargo.lock

@@ -5885,8 +5885,11 @@ dependencies = [
  "ff",
  "group",
  "indexmap 2.1.0",
+ "insta",
  "itertools 0.12.0",
  "jubjub",
+ "proptest",
+ "proptest-derive",
  "rand 0.8.5",
  "semver 1.0.20",
  "serde",

zebra-scan/Cargo.toml

@@ -18,6 +18,21 @@ categories = ["cryptography::cryptocurrencies"]
 
 # Production features that activate extra dependencies, or extra features in dependencies
 
+# Test features
+
+proptest-impl = [
+    "proptest",
+    "proptest-derive",
+    "zebra-state/proptest-impl",
+    "zebra-chain/proptest-impl",
+    "bls12_381",
+    "ff",
+    "group",
+    "jubjub",
+    "rand",
+    "zcash_note_encryption",
+]
+
 [dependencies]
 
 color-eyre = "0.6.2"
@@ -37,15 +52,29 @@ zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = [
 
 chrono = { version = "0.4.31", default-features = false, features = ["clock", "std", "serde"] }
 
+# test feature proptest-impl
+proptest = { version = "1.4.0", optional = true }
+proptest-derive = { version = "0.4.0", optional = true }
+
+bls12_381 = { version = "0.8.0", optional = true }
+ff = { version = "0.13.0", optional = true }
+group = { version = "0.13.0", optional = true }
+jubjub = { version = "0.10.0", optional = true }
+rand = { version = "0.8.5", optional = true }
+zcash_note_encryption = { version = "0.4.0", optional = true }
+
 [dev-dependencies]
 
+insta = { version = "1.33.0", features = ["ron", "redactions"] }
+tokio = { version = "1.34.0", features = ["test-util"] }
+
+proptest = "1.4.0"
+proptest-derive = "0.4.0"
 bls12_381 = "0.8.0"
 ff = "0.13.0"
 group = "0.13.0"
 jubjub = "0.10.0"
 rand = "0.8.5"
-tokio = { version = "1.34.0", features = ["test-util"] }
-
 zcash_note_encryption = "0.4.0"
 
 zebra-state = { path = "../zebra-state", version = "1.0.0-beta.31", features = ["proptest-impl"] }

zebra-scan/src/lib.rs

@@ -9,8 +9,8 @@ pub mod init;
 pub mod scan;
 pub mod storage;
 
-#[cfg(test)]
+#[cfg(any(test, feature = "proptest-impl"))]
-mod tests;
+pub mod tests;
 
 pub use config::Config;
 pub use init::{init, spawn_init};

zebra-scan/src/scan.rs

@@ -211,8 +211,8 @@ pub async fn scan_height_and_store_results(
             let dfvk_res = scanned_block_to_db_result(dfvk_res);
             let ivk_res = scanned_block_to_db_result(ivk_res);
 
-            storage.add_sapling_results(sapling_key.clone(), height, dfvk_res);
+            storage.add_sapling_results(&sapling_key, height, dfvk_res);
-            storage.add_sapling_results(sapling_key, height, ivk_res);
+            storage.add_sapling_results(&sapling_key, height, ivk_res);
 
             Ok::<_, Report>(())
         })
@@ -398,7 +398,7 @@ fn scanned_block_to_db_result<Nf>(
         .map(|tx| {
             (
                 TransactionIndex::from_usize(tx.index),
-                SaplingScannedResult::from(tx.txid.as_ref()),
+                SaplingScannedResult::from_bytes_in_display_order(*tx.txid.as_ref()),
             )
         })
         .collect()

zebra-scan/src/storage.rs

@@ -71,7 +71,13 @@ impl Storage {
     ///
     /// This method can block while writing database files, so it must be inside spawn_blocking()
     /// in async code.
-    pub fn add_sapling_key(&mut self, sapling_key: &SaplingScanningKey, birthday: Option<Height>) {
+    pub fn add_sapling_key(
+        &mut self,
+        sapling_key: &SaplingScanningKey,
+        birthday: impl Into<Option<Height>>,
+    ) {
+        let birthday = birthday.into();
+
         // It's ok to write some keys and not others during shutdown, so each key can get its own
         // batch. (They will be re-written on startup anyway.)
         let mut batch = ScannerWriteBatch::default();
@@ -93,7 +99,8 @@ impl Storage {
         self.sapling_keys_and_birthday_heights()
     }
 
-    /// Add the sapling results for `height` to the storage.
+    /// Add the sapling results for `height` to the storage. The results can be any map of
+    /// [`TransactionIndex`] to [`SaplingScannedResult`].
     ///
     /// # Performance / Hangs
     ///
@@ -101,7 +108,7 @@ impl Storage {
     /// in async code.
     pub fn add_sapling_results(
         &mut self,
-        sapling_key: SaplingScanningKey,
+        sapling_key: &SaplingScanningKey,
         height: Height,
         sapling_results: BTreeMap<TransactionIndex, SaplingScannedResult>,
     ) {

zebra-scan/src/storage/db.rs

@@ -19,6 +19,9 @@ pub use zebra_state::{
 
 pub mod sapling;
 
+#[cfg(test)]
+mod tests;
+
 /// The directory name used to distinguish the scanner database from Zebra's other databases or
 /// flat files.
 ///

zebra-scan/src/storage/db/sapling.rs

@@ -47,13 +47,15 @@ impl Storage {
     // Reading Sapling database entries
 
     /// Returns the result for a specific database index (key, block height, transaction index).
+    /// Returns `None` if the result is missing or an empty marker for a birthday or progress
+    /// height.
     //
     // TODO: add tests for this method
     pub fn sapling_result_for_index(
         &self,
         index: &SaplingScannedDatabaseIndex,
     ) -> Option<SaplingScannedResult> {
-        self.db.zs_get(&self.sapling_tx_ids_cf(), &index)
+        self.db.zs_get(&self.sapling_tx_ids_cf(), &index).flatten()
     }
 
     /// Returns the results for a specific key and block height.

zebra-scan/src/storage/db/tests.rs

@@ -0,0 +1,3 @@
+//! General scanner database tests.
+
+mod snapshot;

zebra-scan/src/storage/db/tests/snapshot.rs

@@ -0,0 +1,164 @@
+//! Raw data snapshot tests for the scanner database format.
+//!
+//! These tests check:
+//! - the name of each column family
+//! - the number of key-value entries
+//! - the bytes in each key and value
+//!
+//! These tests currently use fixed test vectors.
+//!
+//! # Fixing Test Failures
+//!
+//! If this test fails, run:
+//! ```sh
+//! cd zebra-scan
+//! cargo insta test --review --features shielded-scan
+//! ```
+//! to update the test snapshots, then commit the `test_*.snap` files using git.
+//!
+//! # Snapshot Format
+//!
+//! These snapshots use [RON (Rusty Object Notation)](https://github.com/ron-rs/ron#readme),
+//! a text format similar to Rust syntax. Raw byte data is encoded in hexadecimal.
+//!
+//! Due to `serde` limitations, some object types can't be represented exactly,
+//! so RON uses the closest equivalent structure.
+
+use std::{collections::BTreeMap, sync::Arc};
+
+use zebra_chain::{
+    block::{Block, Height},
+    parameters::Network::{self, *},
+    serialization::ZcashDeserializeInto,
+};
+use zebra_state::{RawBytes, ReadDisk, TransactionIndex, KV};
+
+use crate::{
+    storage::{db::ScannerDb, Storage},
+    tests::{FAKE_SAPLING_VIEWING_KEY, ZECPAGES_SAPLING_VIEWING_KEY},
+    Config,
+};
+
+/// Snapshot test for RocksDB column families, and their key-value data.
+///
+/// These snapshots contain the `default` column family, but it is not used by Zebra.
+#[test]
+fn test_raw_rocksdb_column_families() {
+    let _init_guard = zebra_test::init();
+
+    test_raw_rocksdb_column_families_with_network(Mainnet);
+    test_raw_rocksdb_column_families_with_network(Testnet);
+}
+
+/// Snapshot raw column families for `network`.
+///
+/// See [`test_raw_rocksdb_column_families`].
+fn test_raw_rocksdb_column_families_with_network(network: Network) {
+    let mut net_suffix = network.to_string();
+    net_suffix.make_ascii_lowercase();
+
+    let mut storage = Storage::new(&Config::ephemeral(), network);
+
+    // Snapshot the column family names
+    let mut cf_names = storage.db.list_cf().expect("empty database is valid");
+
+    // The order that RocksDB returns column families is irrelevant,
+    // because we always access them by name.
+    cf_names.sort();
+
+    // Assert that column family names are the same, regardless of the network.
+    // Later, we check they are also the same regardless of the block height.
+    insta::assert_ron_snapshot!("column_family_names", cf_names);
+
+    // Assert that empty databases are the same, regardless of the network.
+    let mut settings = insta::Settings::clone_current();
+
+    settings.set_snapshot_suffix("empty");
+    settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
+
+    // Snapshot a birthday that is automatically set to activation height
+    storage.add_sapling_key(&ZECPAGES_SAPLING_VIEWING_KEY.to_string(), None);
+    // Snapshot a birthday above activation height
+    storage.add_sapling_key(&FAKE_SAPLING_VIEWING_KEY.to_string(), Height(1_000_000));
+
+    settings.set_snapshot_suffix(format!("{net_suffix}_keys"));
+    settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
+
+    // Snapshot raw database data for:
+    // - mainnet and testnet
+    // - genesis, block 1, and block 2
+    let blocks = match network {
+        Mainnet => &*zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS,
+        Testnet => &*zebra_test::vectors::CONTINUOUS_TESTNET_BLOCKS,
+    };
+
+    // We limit the number of blocks, because the serialized data is a few kilobytes per block.
+    for height in 0..=2 {
+        let block: Arc<Block> = blocks
+            .get(&height)
+            .expect("block height has test data")
+            .zcash_deserialize_into()
+            .expect("test data deserializes");
+
+        // Fake results from the first few blocks
+        storage.add_sapling_results(
+            &ZECPAGES_SAPLING_VIEWING_KEY.to_string(),
+            Height(height),
+            block
+                .transactions
+                .iter()
+                .enumerate()
+                .map(|(index, tx)| (TransactionIndex::from_usize(index), tx.hash().into()))
+                .collect(),
+        );
+
+        let mut settings = insta::Settings::clone_current();
+        settings.set_snapshot_suffix(format!("{net_suffix}_{height}"));
+
+        settings.bind(|| snapshot_raw_rocksdb_column_family_data(&storage.db, &cf_names));
+    }
+}
+
+/// Snapshot the data in each column family, using `cargo insta` and RON serialization.
+fn snapshot_raw_rocksdb_column_family_data(db: &ScannerDb, original_cf_names: &[String]) {
+    let mut new_cf_names = db.list_cf().expect("empty database is valid");
+    new_cf_names.sort();
+
+    // Assert that column family names are the same, regardless of the network or block height.
+    assert_eq!(
+        original_cf_names, new_cf_names,
+        "unexpected extra column families",
+    );
+
+    let mut empty_column_families = Vec::new();
+
+    // Now run the data snapshots
+    for cf_name in original_cf_names {
+        let cf_handle = db
+            .cf_handle(cf_name)
+            .expect("RocksDB API provides correct names");
+
+        // Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests.
+        let cf_items: BTreeMap<RawBytes, RawBytes> = db.zs_items_in_range_ordered(&cf_handle, ..);
+
+        // The default raw data serialization is very verbose, so we hex-encode the bytes.
+        let cf_data: Vec<KV> = cf_items
+            .iter()
+            .map(|(key, value)| KV::new(key.raw_bytes(), value.raw_bytes()))
+            .collect();
+
+        if cf_name == "default" {
+            assert_eq!(cf_data.len(), 0, "default column family is never used");
+        } else if cf_data.is_empty() {
+            // distinguish column family names from empty column families
+            empty_column_families.push(format!("{cf_name}: no entries"));
+        } else {
+            // The note commitment tree snapshots will change if the trees do not have cached roots.
+            // But we expect them to always have cached roots,
+            // because those roots are used to populate the anchor column families.
+            insta::assert_ron_snapshot!(format!("{cf_name}_raw_data"), cf_data);
+        }
+    }
+
+    insta::assert_ron_snapshot!("empty_column_families", empty_column_families);
+}

zebra-scan/src/storage/db/tests/snapshots/column_family_names.snap

@@ -0,0 +1,8 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_names
+---
+[
+  "default",
+  "sapling_tx_ids",
+]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@empty.snap

@@ -0,0 +1,7 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[
+  "sapling_tx_ids: no entries",
+]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_0.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_1.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_2.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@mainnet_keys.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_0.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_1.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_2.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/empty_column_families@testnet_keys.snap

@@ -0,0 +1,5 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: empty_column_families
+---
+[]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_0.snap

@@ -0,0 +1,18 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
+    v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_1.snap

@@ -0,0 +1,22 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
+    v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
+    v: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_2.snap

@@ -0,0 +1,26 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
+    v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
+    v: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000020000",
+    v: "8974d08d1c5f9c860d8b629d582a56659a4a1dcb2b5f98a25a5afcc2a784b0f4",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@mainnet_keys.snap

@@ -0,0 +1,14 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a06657f0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_0.snap

@@ -0,0 +1,18 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
+    v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_1.snap

@@ -0,0 +1,22 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
+    v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
+    v: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_2.snap

@@ -0,0 +1,26 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000000000",
+    v: "c4eaa58879081de3c24a7b117ed2b28300e7ec4c4c1dff1d3f1268b7857a4ddb",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000010000",
+    v: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0000020000",
+    v: "5822c0532da8a008259ac39933d3210e508c17e3ba21d2b2c428785efdccb3d5",
+  ),
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/storage/db/tests/snapshots/sapling_tx_ids_raw_data@testnet_keys.snap

@@ -0,0 +1,14 @@
+---
+source: zebra-scan/src/storage/db/tests/snapshot.rs
+expression: cf_data
+---
+[
+  KV(
+    k: "7a78766965777331713064757974676371717171707172653236776b6c343567767777776437303678773630386875636d7666616c72373539656a7766377173686a663572396161373332337a756c767a36706c68747470356d6c747163677339743033396378326430396d67713035747336336e387533356879763668396e633963747171747565327537636572326d716567756e75756c71326c7568713379776a637a333579796c6a657761346d676b676a7a79667768366672366a6430647a64343467686b306e78647632686e76346a356e7866777632347277646d676c6c68653070383536387367717439636b74303276326b786635616874716c3673306c746a706b636b77386774796d787478757539676372307377767a0445bf0000",
+    v: "",
+  ),
+  KV(
+    k: "7a78766965777366616b650f423f0000",
+    v: "",
+  ),
+]

zebra-scan/src/tests.rs

@@ -12,17 +12,14 @@ use ff::{Field, PrimeField};
 use group::GroupEncoding;
 use rand::{rngs::OsRng, thread_rng, RngCore};
 
-use zcash_client_backend::{
+use zcash_client_backend::proto::compact_formats::{
-    encoding::decode_extended_full_viewing_key,
+    ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
-    proto::compact_formats::{
-        ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
-    },
 };
 use zcash_note_encryption::Domain;
 use zcash_primitives::{
     block::BlockHash,
     consensus::BlockHeight,
-    constants::{mainnet::HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, SPENDING_KEY_GENERATOR},
+    constants::SPENDING_KEY_GENERATOR,
     memo::MemoBytes,
     sapling::{
         note_encryption::{sapling_note_encryption, SaplingDomain},
@@ -30,200 +27,29 @@ use zcash_primitives::{
         value::NoteValue,
         Note, Nullifier,
     },
-    zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey},
+    zip32::DiversifiableFullViewingKey,
 };
 
 use zebra_chain::{
     amount::{Amount, NegativeAllowed},
     block::{self, merkle, Block, Header, Height},
-    chain_tip::ChainTip,
     fmt::HexDebug,
-    parameters::Network,
     primitives::{redjubjub, Groth16Proof},
     sapling::{self, PerSpendAnchor, Spend, TransferData},
-    serialization::{AtLeastOne, ZcashDeserializeInto},
+    serialization::AtLeastOne,
     transaction::{LockTime, Transaction},
     transparent::{CoinbaseData, Input},
     work::{difficulty::CompactDifficulty, equihash::Solution},
 };
-use zebra_state::{SaplingScannedResult, TransactionIndex};
 
-use crate::{
+#[cfg(test)]
-    config::Config,
+mod vectors;
-    scan::{block_to_compact, scan_block},
-};
 
-/// This test:
+/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
-/// - Creates a viewing key and a fake block containing a Sapling output decryptable by the key.
+pub const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
-/// - Scans the block.
-/// - Checks that the result contains the txid of the tx containing the Sapling output.
-#[tokio::test]
-async fn scanning_from_fake_generated_blocks() -> Result<()> {
-    let extsk = ExtendedSpendingKey::master(&[]);
-    let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
-    let nf = Nullifier([7; 32]);
 
-    let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
+/// A fake viewing key in an incorrect format.
-
+pub const FAKE_SAPLING_VIEWING_KEY: &str = "zxviewsfake";
-    assert_eq!(block.transactions.len(), 4);
-
-    let res = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
-
-    // The response should have one transaction relevant to the key we provided.
-    assert_eq!(res.transactions().len(), 1);
-
-    // Check that the original block contains the txid in the scanning result.
-    assert!(block
-        .transactions
-        .iter()
-        .map(|tx| tx.hash().bytes_in_display_order())
-        .any(|txid| &txid == res.transactions()[0].txid.as_ref()));
-
-    // Check that the txid in the scanning result matches the third tx in the original block.
-    assert_eq!(
-        res.transactions()[0].txid.as_ref(),
-        &block.transactions[2].hash().bytes_in_display_order()
-    );
-
-    // The block hash of the response should be the same as the one provided.
-    assert_eq!(res.block_hash().0, block.hash().0);
-
-    Ok(())
-}
-
-/// Scan a populated state for the ZECpages viewing key.
-/// This test is very similar to `scanning_from_populated_zebra_state` but with the ZECpages key.
-/// There are no zechub transactions in the test data so we should get empty related transactions.
-#[tokio::test]
-async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
-    /// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
-    const ZECPAGES_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
-
-    // Parse the key from ZECpages
-    let efvk = decode_extended_full_viewing_key(
-        HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY,
-        ZECPAGES_VIEWING_KEY,
-    )
-    .unwrap();
-
-    // Build a vector of viewing keys `vks` to scan for.
-    let fvk = efvk.fvk;
-    let ivk = fvk.vk.ivk();
-    let ivks = vec![ivk];
-
-    let network = Network::Mainnet;
-
-    // Create a continuous chain of mainnet blocks from genesis
-    let blocks: Vec<Arc<Block>> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS
-        .iter()
-        .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap())
-        .collect();
-
-    // Create a populated state service.
-    let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) =
-        zebra_state::populated_state(blocks.clone(), network).await;
-
-    let db = read_only_state_service.db();
-
-    // use the tip as starting height
-    let mut height = latest_chain_tip.best_tip_height().unwrap();
-
-    let mut transactions_found = 0;
-    let mut transactions_scanned = 0;
-    let mut blocks_scanned = 0;
-    while let Some(block) = db.block(height.into()) {
-        // We use a dummy size of the Sapling note commitment tree. We can't set the size to zero
-        // because the underlying scanning function would return
-        // `zcash_client_backeng::scanning::ScanError::TreeSizeUnknown`.
-        let sapling_commitment_tree_size = 1;
-
-        let orchard_commitment_tree_size = 0;
-
-        let chain_metadata = ChainMetadata {
-            sapling_commitment_tree_size,
-            orchard_commitment_tree_size,
-        };
-
-        let compact_block = block_to_compact(&block, chain_metadata);
-
-        let res = scan_block(network, &block, sapling_commitment_tree_size, &ivks)
-            .expect("scanning block for the ZECpages viewing key should work");
-
-        transactions_found += res.transactions().len();
-        transactions_scanned += compact_block.vtx.len();
-        blocks_scanned += 1;
-
-        // scan backwards
-        if height.is_min() {
-            break;
-        }
-        height = height.previous()?;
-    }
-
-    // make sure all blocks and transactions were scanned
-    assert_eq!(blocks_scanned, 11);
-    assert_eq!(transactions_scanned, 11);
-
-    // no relevant transactions should be found
-    assert_eq!(transactions_found, 0);
-
-    Ok(())
-}
-
-/// Creates a viewing key and a fake block containing a Sapling output decryptable by the key, scans
-/// the block using the key, and adds the results to the database.
-///
-/// The purpose of this test is to check if our database and our scanning code are compatible.
-#[test]
-#[allow(deprecated)]
-fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
-    // Generate a key
-    let extsk = ExtendedSpendingKey::master(&[]);
-    // TODO: find out how to do it with `to_diversifiable_full_viewing_key` as `to_extended_full_viewing_key` is deprecated.
-    let extfvk = extsk.to_extended_full_viewing_key();
-    let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
-    let key_to_be_stored =
-        zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk);
-
-    // Create a database
-    let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet);
-
-    // Insert the generated key to the database
-    s.add_sapling_key(&key_to_be_stored, None);
-
-    // Check key was added
-    assert_eq!(s.sapling_keys().len(), 1);
-    assert_eq!(
-        s.sapling_keys().get(&key_to_be_stored),
-        Some(&s.min_sapling_birthday_height())
-    );
-
-    let nf = Nullifier([7; 32]);
-
-    let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
-
-    let result = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
-
-    // The response should have one transaction relevant to the key we provided.
-    assert_eq!(result.transactions().len(), 1);
-
-    let result = SaplingScannedResult::from(result.transactions()[0].txid.as_ref());
-
-    // Add result to database
-    s.add_sapling_results(
-        key_to_be_stored.clone(),
-        Height(1),
-        [(TransactionIndex::from_usize(0), result)].into(),
-    );
-
-    // Check the result was added
-    assert_eq!(
-        s.sapling_results(&key_to_be_stored).get(&Height(1)),
-        Some(&vec![result])
-    );
-
-    Ok(())
-}
 
 /// Generates a fake block containing a Sapling output decryptable by `dfvk`.
 ///
@@ -232,7 +58,7 @@ fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
 /// 2. a V4 tx containing a random Sapling output,
 /// 3. a V4 tx containing a Sapling output decryptable by `dfvk`,
 /// 4. depending on the value of `tx_after`, another V4 tx containing a random Sapling output.
-fn fake_block(
+pub fn fake_block(
     height: BlockHeight,
     nf: Nullifier,
     dfvk: &DiversifiableFullViewingKey,
@@ -305,7 +131,7 @@ fn fake_block(
 // be a number for easier conversion:
 // https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L635
 // We need to copy because this is a test private function upstream.
-fn fake_compact_block(
+pub fn fake_compact_block(
     height: BlockHeight,
     prev_hash: BlockHash,
     nf: Nullifier,
@@ -394,7 +220,7 @@ fn fake_compact_block(
 // This is an exact copy of `zcash_client_backend::scanning::random_compact_tx`:
 // https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L597
 // We need to copy because this is a test private function upstream.
-fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
+pub fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
     let fake_nf = {
         let mut nf = vec![0; 32];
         rng.fill_bytes(&mut nf);
@@ -427,7 +253,7 @@ fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
 }
 
 /// Converts [`CompactTx`] to [`Transaction::V4`].
-fn compact_to_v4(tx: &CompactTx) -> Result<Transaction> {
+pub fn compact_to_v4(tx: &CompactTx) -> Result<Transaction> {
     let sk = redjubjub::SigningKey::<redjubjub::SpendAuth>::new(thread_rng());
     let vk = redjubjub::VerificationKey::from(&sk);
     let dummy_rk = sapling::keys::ValidatingKey::try_from(vk)

zebra-scan/src/tests/vectors.rs

@@ -0,0 +1,198 @@
+//! Fixed integration test vectors for the scanner.
+
+use std::sync::Arc;
+
+use color_eyre::Result;
+
+use zcash_client_backend::{
+    encoding::decode_extended_full_viewing_key, proto::compact_formats::ChainMetadata,
+};
+use zcash_primitives::{
+    constants::mainnet::HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY,
+    sapling::Nullifier,
+    zip32::{DiversifiableFullViewingKey, ExtendedSpendingKey},
+};
+
+use zebra_chain::{
+    block::{Block, Height},
+    chain_tip::ChainTip,
+    parameters::Network,
+    serialization::ZcashDeserializeInto,
+};
+use zebra_state::{SaplingScannedResult, TransactionIndex};
+
+use crate::{
+    config::Config,
+    scan::{block_to_compact, scan_block},
+    tests::{fake_block, ZECPAGES_SAPLING_VIEWING_KEY},
+};
+
+/// This test:
+/// - Creates a viewing key and a fake block containing a Sapling output decryptable by the key.
+/// - Scans the block.
+/// - Checks that the result contains the txid of the tx containing the Sapling output.
+#[tokio::test]
+async fn scanning_from_fake_generated_blocks() -> Result<()> {
+    let extsk = ExtendedSpendingKey::master(&[]);
+    let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
+    let nf = Nullifier([7; 32]);
+
+    let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
+
+    assert_eq!(block.transactions.len(), 4);
+
+    let res = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
+
+    // The response should have one transaction relevant to the key we provided.
+    assert_eq!(res.transactions().len(), 1);
+
+    // Check that the original block contains the txid in the scanning result.
+    assert!(block
+        .transactions
+        .iter()
+        .map(|tx| tx.hash().bytes_in_display_order())
+        .any(|txid| &txid == res.transactions()[0].txid.as_ref()));
+
+    // Check that the txid in the scanning result matches the third tx in the original block.
+    assert_eq!(
+        res.transactions()[0].txid.as_ref(),
+        &block.transactions[2].hash().bytes_in_display_order()
+    );
+
+    // The block hash of the response should be the same as the one provided.
+    assert_eq!(res.block_hash().0, block.hash().0);
+
+    Ok(())
+}
+
+/// Scan a populated state for the ZECpages viewing key.
+/// This test is very similar to `scanning_from_populated_zebra_state` but with the ZECpages key.
+/// There are no zechub transactions in the test data so we should get empty related transactions.
+#[tokio::test]
+async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> {
+    // Parse the key from ZECpages
+    let efvk = decode_extended_full_viewing_key(
+        HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY,
+        ZECPAGES_SAPLING_VIEWING_KEY,
+    )
+    .unwrap();
+
+    // Build a vector of viewing keys `vks` to scan for.
+    let fvk = efvk.fvk;
+    let ivk = fvk.vk.ivk();
+    let ivks = vec![ivk];
+
+    let network = Network::Mainnet;
+
+    // Create a continuous chain of mainnet blocks from genesis
+    let blocks: Vec<Arc<Block>> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS
+        .iter()
+        .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap())
+        .collect();
+
+    // Create a populated state service.
+    let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) =
+        zebra_state::populated_state(blocks.clone(), network).await;
+
+    let db = read_only_state_service.db();
+
+    // use the tip as starting height
+    let mut height = latest_chain_tip.best_tip_height().unwrap();
+
+    let mut transactions_found = 0;
+    let mut transactions_scanned = 0;
+    let mut blocks_scanned = 0;
+    while let Some(block) = db.block(height.into()) {
+        // We use a dummy size of the Sapling note commitment tree. We can't set the size to zero
+        // because the underlying scanning function would return
+        // `zcash_client_backeng::scanning::ScanError::TreeSizeUnknown`.
+        let sapling_commitment_tree_size = 1;
+
+        let orchard_commitment_tree_size = 0;
+
+        let chain_metadata = ChainMetadata {
+            sapling_commitment_tree_size,
+            orchard_commitment_tree_size,
+        };
+
+        let compact_block = block_to_compact(&block, chain_metadata);
+
+        let res = scan_block(network, &block, sapling_commitment_tree_size, &ivks)
+            .expect("scanning block for the ZECpages viewing key should work");
+
+        transactions_found += res.transactions().len();
+        transactions_scanned += compact_block.vtx.len();
+        blocks_scanned += 1;
+
+        // scan backwards
+        if height.is_min() {
+            break;
+        }
+        height = height.previous()?;
+    }
+
+    // make sure all blocks and transactions were scanned
+    assert_eq!(blocks_scanned, 11);
+    assert_eq!(transactions_scanned, 11);
+
+    // no relevant transactions should be found
+    assert_eq!(transactions_found, 0);
+
+    Ok(())
+}
+
+/// Creates a viewing key and a fake block containing a Sapling output decryptable by the key, scans
+/// the block using the key, and adds the results to the database.
+///
+/// The purpose of this test is to check if our database and our scanning code are compatible.
+#[test]
+#[allow(deprecated)]
+fn scanning_fake_blocks_store_key_and_results() -> Result<()> {
+    // Generate a key
+    let extsk = ExtendedSpendingKey::master(&[]);
+    // TODO: find out how to do it with `to_diversifiable_full_viewing_key` as `to_extended_full_viewing_key` is deprecated.
+    let extfvk = extsk.to_extended_full_viewing_key();
+    let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key();
+    let key_to_be_stored =
+        zcash_client_backend::encoding::encode_extended_full_viewing_key("zxviews", &extfvk);
+
+    // Create a database
+    let mut s = crate::storage::Storage::new(&Config::ephemeral(), Network::Mainnet);
+
+    // Insert the generated key to the database
+    s.add_sapling_key(&key_to_be_stored, None);
+
+    // Check key was added
+    assert_eq!(s.sapling_keys().len(), 1);
+    assert_eq!(
+        s.sapling_keys().get(&key_to_be_stored),
+        Some(&s.min_sapling_birthday_height())
+    );
+
+    let nf = Nullifier([7; 32]);
+
+    let (block, sapling_tree_size) = fake_block(1u32.into(), nf, &dfvk, 1, true, Some(0));
+
+    let result = scan_block(Network::Mainnet, &block, sapling_tree_size, &[&dfvk]).unwrap();
+
+    // The response should have one transaction relevant to the key we provided.
+    assert_eq!(result.transactions().len(), 1);
+
+    let result =
+        SaplingScannedResult::from_bytes_in_display_order(*result.transactions()[0].txid.as_ref());
+
+    // Add result to database
+    s.add_sapling_results(
+        &key_to_be_stored,
+        Height(1),
+        [(TransactionIndex::from_usize(0), result)].into(),
+    );
+
+    // Check the result was added
+    assert_eq!(
+        s.sapling_results(&key_to_be_stored).get(&Height(1)),
+        Some(&vec![result])
+    );
+
+    Ok(())
+}

zebra-state/src/lib.rs

@@ -63,12 +63,12 @@ pub use service::{
 pub use rocksdb::AsColumnFamilyRef;
 #[cfg(feature = "shielded-scan")]
 pub use service::finalized_state::{
-    FromDisk, IntoDisk, ReadDisk, SaplingScannedDatabaseEntry, SaplingScannedDatabaseIndex,
+    FromDisk, IntoDisk, SaplingScannedDatabaseEntry, SaplingScannedDatabaseIndex,
     SaplingScannedResult, SaplingScanningKey, ZebraDb,
 };
 
 #[cfg(any(test, feature = "proptest-impl", feature = "shielded-scan"))]
-pub use service::finalized_state::{DiskWriteBatch, WriteDisk};
+pub use service::finalized_state::{DiskWriteBatch, ReadDisk, WriteDisk};
 
 #[cfg(feature = "getblocktemplate-rpcs")]
 pub use response::GetBlockTemplateChainInfo;
@@ -77,7 +77,7 @@ pub use response::GetBlockTemplateChainInfo;
 pub use service::{
     arbitrary::{populated_state, CHAIN_TIP_UPDATE_WAIT_LIMIT},
     chain_tip::{ChainTipBlock, ChainTipSender},
-    finalized_state::MAX_ON_DISK_HEIGHT,
+    finalized_state::{RawBytes, KV, MAX_ON_DISK_HEIGHT},
     init_test, init_test_services, ReadStateService,
 };
 

zebra-state/src/service/finalized_state.rs

@@ -42,8 +42,8 @@ mod tests;
 pub use disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk};
 #[allow(unused_imports)]
 pub use disk_format::{
-    FromDisk, IntoDisk, OutputIndex, OutputLocation, TransactionIndex, TransactionLocation,
+    FromDisk, IntoDisk, OutputIndex, OutputLocation, RawBytes, TransactionIndex,
-    MAX_ON_DISK_HEIGHT,
+    TransactionLocation, MAX_ON_DISK_HEIGHT,
 };
 pub use zebra_db::ZebraDb;
 
@@ -53,6 +53,9 @@ pub use disk_format::{
     SaplingScanningKey,
 };
 
+#[cfg(any(test, feature = "proptest-impl"))]
+pub use disk_format::KV;
+
 /// The column families supported by the running `zebra-state` database code.
 ///
 /// Existing column families that aren't listed here are preserved when the database is opened.

zebra-state/src/service/finalized_state/disk_format.rs

@@ -16,7 +16,7 @@ pub mod upgrade;
 #[cfg(feature = "shielded-scan")]
 pub mod scan;
 
-#[cfg(test)]
+#[cfg(any(test, feature = "proptest-impl"))]
 mod tests;
 
 pub use block::{TransactionIndex, TransactionLocation, MAX_ON_DISK_HEIGHT};
@@ -28,6 +28,9 @@ pub use scan::{
     SaplingScanningKey,
 };
 
+#[cfg(any(test, feature = "proptest-impl"))]
+pub use tests::KV;
+
 /// Helper type for writing types to disk as raw bytes.
 /// Also used to convert key types to raw bytes for disk lookups.
 pub trait IntoDisk {

zebra-state/src/service/finalized_state/disk_format/scan.rs

@@ -37,9 +37,21 @@ impl From<SaplingScannedResult> for transaction::Hash {
     }
 }
 
-impl From<&[u8; 32]> for SaplingScannedResult {
+impl From<transaction::Hash> for SaplingScannedResult {
-    fn from(bytes: &[u8; 32]) -> Self {
+    fn from(hash: transaction::Hash) -> Self {
-        Self(*bytes)
+        SaplingScannedResult(hash.bytes_in_display_order())
+    }
+}
+
+impl SaplingScannedResult {
+    /// Creates a `SaplingScannedResult` from bytes in display order.
+    pub fn from_bytes_in_display_order(bytes: [u8; 32]) -> Self {
+        Self(bytes)
+    }
+
+    /// Returns the inner bytes in display order.
+    pub fn bytes_in_display_order(&self) -> [u8; 32] {
+        self.0
     }
 }
 
@@ -162,19 +174,8 @@ impl FromDisk for SaplingScannedDatabaseIndex {
     }
 }
 
-impl IntoDisk for SaplingScannedResult {
+// We can't implement IntoDisk or FromDisk for SaplingScannedResult,
-    type Bytes = [u8; 32];
+// because the format is actually Option<SaplingScannedResult>.
-
-    fn as_bytes(&self) -> Self::Bytes {
-        self.0
-    }
-}
-
-impl FromDisk for SaplingScannedResult {
-    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
-        SaplingScannedResult(bytes.as_ref().try_into().unwrap())
-    }
-}
 
 impl IntoDisk for Option<SaplingScannedResult> {
     type Bytes = Vec<u8>;
@@ -183,7 +184,7 @@ impl IntoDisk for Option<SaplingScannedResult> {
         let mut bytes = Vec::new();
 
         if let Some(result) = self.as_ref() {
-            bytes.extend(result.as_bytes());
+            bytes.extend(result.bytes_in_display_order());
         }
 
         bytes
@@ -191,13 +192,18 @@ impl IntoDisk for Option<SaplingScannedResult> {
 }
 
 impl FromDisk for Option<SaplingScannedResult> {
+    #[allow(clippy::unwrap_in_result)]
     fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
         let bytes = bytes.as_ref();
 
         if bytes.is_empty() {
             None
         } else {
-            Some(SaplingScannedResult::from_bytes(bytes))
+            Some(SaplingScannedResult::from_bytes_in_display_order(
+                bytes
+                    .try_into()
+                    .expect("unexpected incorrect SaplingScannedResult data length"),
+            ))
         }
     }
 }

zebra-state/src/service/finalized_state/disk_format/scan/tests/prop.rs

@@ -29,13 +29,6 @@ fn roundtrip_sapling_db_index() {
     );
 }
 
-#[test]
-fn roundtrip_sapling_result() {
-    let _init_guard = zebra_test::init();
-
-    proptest!(|(val in any::<SaplingScannedResult>())| assert_value_properties(val));
-}
-
 #[test]
 fn roundtrip_option_sapling_result() {
     let _init_guard = zebra_test::init();

zebra-state/src/service/finalized_state/disk_format/tests.rs

@@ -2,12 +2,14 @@
 
 use serde::{Deserialize, Serialize};
 
+#[cfg(test)]
 mod prop;
+#[cfg(test)]
 mod snapshot;
 
 /// A formatting struct for raw key-value data
 #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
-struct KV {
+pub struct KV {
     /// The raw key bytes, as a hexadecimal-encoded string.
     k: String,
 
@@ -17,7 +19,7 @@ struct KV {
 
 impl KV {
     /// Create a new `KV` from raw key-value data.
-    fn new<K, V>(key: K, value: V) -> KV
+    pub fn new<K, V>(key: K, value: V) -> KV
     where
         K: AsRef<[u8]>,
         V: AsRef<[u8]>,
@@ -26,7 +28,7 @@ impl KV {
     }
 
     /// Create a new `KV` from hex-encoded key-value data.
-    fn new_hex(key: String, value: String) -> KV {
+    pub fn new_hex(key: String, value: String) -> KV {
         KV { k: key, v: value }
     }
 }

zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs

@@ -27,7 +27,7 @@
 //!
 //! Test shielded data, and data activated in Overwinter and later network upgrades.
 
-use std::sync::Arc;
+use std::{collections::BTreeMap, sync::Arc};
 
 use zebra_chain::{
     block::Block,
@@ -37,11 +37,11 @@ use zebra_chain::{
 
 use crate::{
     service::finalized_state::{
-        disk_db::{DiskDb, DB},
+        disk_db::DiskDb,
-        disk_format::tests::KV,
+        disk_format::{tests::KV, RawBytes},
         FinalizedState,
     },
-    Config,
+    Config, ReadDisk,
 };
 
 /// Snapshot test for RocksDB column families, and their key-value data.
@@ -133,13 +133,12 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
             .expect("RocksDB API provides correct names");
 
         // Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests.
-        let mut cf_iter = db.full_iterator_cf(&cf_handle, rocksdb::IteratorMode::Start);
+        let cf_items: BTreeMap<RawBytes, RawBytes> = db.zs_items_in_range_ordered(&cf_handle, ..);
 
         // The default raw data serialization is very verbose, so we hex-encode the bytes.
-        let cf_data: Vec<KV> = cf_iter
+        let cf_data: Vec<KV> = cf_items
-            .by_ref()
+            .iter()
-            .map(|result| result.expect("unexpected database error"))
+            .map(|(key, value)| KV::new(key.raw_bytes(), value.raw_bytes()))
-            .map(|(key, value)| KV::new(key, value))
             .collect();
 
         if cf_name == "default" {
@@ -153,14 +152,6 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
             // because those roots are used to populate the anchor column families.
             insta::assert_ron_snapshot!(format!("{cf_name}_raw_data"), cf_data);
         }
-
-        let raw_cf_iter: rocksdb::DBRawIteratorWithThreadMode<DB> = cf_iter.into();
-
-        assert_eq!(
-            raw_cf_iter.status(),
-            Ok(()),
-            "unexpected column family iterator error",
-        );
     }
 
     insta::assert_ron_snapshot!("empty_column_families", empty_column_families);

zebrad/Cargo.toml

@@ -118,6 +118,7 @@ proptest-impl = [
     "zebra-state/proptest-impl",
     "zebra-network/proptest-impl",
     "zebra-chain/proptest-impl",
+    "zebra-scan?/proptest-impl",
 ]
 
 # Build the zebra-checkpoints utility for checkpoint generation tests
@@ -271,6 +272,7 @@ zebra-chain = { path = "../zebra-chain", features = ["proptest-impl"] }
 zebra-consensus = { path = "../zebra-consensus", features = ["proptest-impl"] }
 zebra-network = { path = "../zebra-network", features = ["proptest-impl"] }
 zebra-state = { path = "../zebra-state", features = ["proptest-impl"] }
+zebra-scan = { path = "../zebra-scan", features = ["proptest-impl"] }
 
 zebra-node-services = { path = "../zebra-node-services", features = ["rpc-client"] }
 

zebrad/tests/acceptance.rs

@@ -2821,13 +2821,13 @@ async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> {
 fn scan_task_starts() -> Result<()> {
     use indexmap::IndexMap;
 
-    const ZECPAGES_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";
+    use zebra_scan::tests::ZECPAGES_SAPLING_VIEWING_KEY;
 
     let _init_guard = zebra_test::init();
 
     let mut config = default_test_config(Mainnet)?;
     let mut keys = IndexMap::new();
-    keys.insert(ZECPAGES_VIEWING_KEY.to_string(), 1);
+    keys.insert(ZECPAGES_SAPLING_VIEWING_KEY.to_string(), 1);
     config.shielded_scan.sapling_keys_to_scan = keys;
 
     let testdir = testdir()?.with_config(&mut config)?;