tests(rpc): Add fixed test vectors for `z_getsubtreesbyindex` from zcashd to zebra (#7515)

* add fixed test vectors for `z_getsubtreesbyindex`

* change to snapshots

* add test to docker

* remove assert lines from snapshots

* add more tests

* change test description

* change test name

* run both tests together

* wait for state version update in test

* Run one test at a time to avoid state locking issues

---------

Co-authored-by: teor <teor@riseup.net>

Cargo.lock

@@ -5839,6 +5839,7 @@ dependencies = [
  "indexmap 2.0.0",
  "indicatif",
  "inferno",
+ "insta",
  "jsonrpc-core",
  "lazy_static",
  "log",

docker/entrypoint.sh

@@ -85,7 +85,9 @@ case "$1" in
         elif [[ "$TEST_LWD_RPC_CALL" -eq "1" ]]; then
             # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
-            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored fully_synced_rpc_test
+            # Run both the fully synced RPC test and the subtree snapshot test, one test at a time.
+            # Since these tests use the same cached state, a state problem in the first test can fail the second test.
+            cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored --test-threads 1 fully_synced_rpc_
         elif [[ "$TEST_LWD_FULL_SYNC" -eq "1" ]]; then
             # Starting at a cached Zebra tip, run a lightwalletd sync to tip.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")

zebrad/Cargo.toml

@@ -235,6 +235,7 @@ hex-literal = "0.4.1"
 jsonrpc-core = "18.0.0"
 once_cell = "1.18.0"
 regex = "1.9.5"
+insta = { version = "1.31.0", features = ["json"] }
 
 # zebra-rpc needs the preserve_order feature, it also makes test results more stable
 serde_json = { version = "1.0.107", features = ["preserve_order"] }

zebrad/tests/acceptance.rs

@@ -151,6 +151,7 @@ use color_eyre::{
     Help,
 };
 use semver::Version;
+use serde_json::Value;
 
 use zebra_chain::{
     block::{self, Height},
@@ -1420,8 +1421,6 @@ async fn rpc_endpoint_parallel_threads() -> Result<()> {
 /// Set `parallel_cpu_threads` to true to auto-configure based on the number of CPU cores.
 #[tracing::instrument]
 async fn rpc_endpoint(parallel_cpu_threads: bool) -> Result<()> {
-    use serde_json::Value;
-
     let _init_guard = zebra_test::init();
     if zebra_test::net::zebra_skip_network_tests() {
         return Ok(());
@@ -2636,3 +2635,103 @@ async fn state_format_test(
     }
     Ok(())
 }
+
+/// Snapshot the `z_getsubtreesbyindex` method in a synchronized chain.
+///
+/// This test name must have the same prefix as the `fully_synced_rpc_test`, so they can be run in the same test job.
+#[tokio::test]
+#[ignore]
+async fn fully_synced_rpc_z_getsubtreesbyindex_snapshot_test() -> Result<()> {
+    let _init_guard = zebra_test::init();
+
+    // We're only using cached Zebra state here, so this test type is the most similar
+    let test_type = TestType::UpdateZebraCachedStateWithRpc;
+    let network = Network::Mainnet;
+
+    let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = spawn_zebrad_for_rpc(
+        network,
+        "rpc_z_getsubtreesbyindex_sync_snapshots",
+        test_type,
+        true,
+    )? {
+        tracing::info!("running fully synced zebrad z_getsubtreesbyindex RPC test");
+
+        zebrad_and_address
+    } else {
+        // Skip the test, we don't have the required cached state
+        return Ok(());
+    };
+
+    // Store the state version message so we can wait for the upgrade later if needed.
+    let state_version_message = wait_for_state_version_message(&mut zebrad)?;
+
+    // Wait for zebrad to load the full cached blockchain.
+    zebrad.expect_stdout_line_matches(SYNC_FINISHED_REGEX)?;
+
+    // Create an http client
+    let client =
+        RpcRequestClient::new(zebra_rpc_address.expect("already checked that address is valid"));
+
+    // Create test vector matrix
+    let zcashd_test_vectors = vec![
+        (
+            "z_getsubtreesbyindex_mainnet_sapling_0_1".to_string(),
+            r#"["sapling", 0, 1]"#.to_string(),
+        ),
+        (
+            "z_getsubtreesbyindex_mainnet_sapling_0_11".to_string(),
+            r#"["sapling", 0, 11]"#.to_string(),
+        ),
+        (
+            "z_getsubtreesbyindex_mainnet_sapling_17_1".to_string(),
+            r#"["sapling", 17, 1]"#.to_string(),
+        ),
+        (
+            "z_getsubtreesbyindex_mainnet_sapling_1090_6".to_string(),
+            r#"["sapling", 1090, 6]"#.to_string(),
+        ),
+        (
+            "z_getsubtreesbyindex_mainnet_orchard_0_1".to_string(),
+            r#"["orchard", 0, 1]"#.to_string(),
+        ),
+        (
+            "z_getsubtreesbyindex_mainnet_orchard_338_1".to_string(),
+            r#"["orchard", 338, 1]"#.to_string(),
+        ),
+        (
+            "z_getsubtreesbyindex_mainnet_orchard_585_1".to_string(),
+            r#"["orchard", 585, 1]"#.to_string(),
+        ),
+    ];
+
+    // Before we write a cached state image, wait for a database upgrade.
+    //
+    // TODO: this line will hang if the state upgrade finishes before zebra is synced.
+    // But that is unlikely with the 25.2 upgrade, because it takes 20+ minutes.
+    // If it happens for a later upgrade, this code can be moved earlier in the test,
+    // as long as all the cached states are version 25.2.2 or later.
+    wait_for_state_version_upgrade(
+        &mut zebrad,
+        &state_version_message,
+        database_format_version_in_code(),
+    )?;
+
+    for i in zcashd_test_vectors {
+        let res = client.call("z_getsubtreesbyindex", i.1).await?;
+        let body = res.bytes().await;
+        let parsed: Value = serde_json::from_slice(&body.expect("Response is valid json"))?;
+        insta::assert_json_snapshot!(i.0, parsed);
+    }
+
+    zebrad.kill(false)?;
+
+    let output = zebrad.wait_with_output()?;
+    let output = output.assert_failure()?;
+
+    // [Note on port conflict](#Note on port conflict)
+    output
+        .assert_was_killed()
+        .wrap_err("Possible port conflict. Are there other acceptance tests running?")?;
+
+    Ok(())
+}

zebrad/tests/snapshots/z_getsubtreesbyindex_mainnet_orchard_0_1.snap

@@ -0,0 +1,18 @@
+---
+source: zebrad/tests/acceptance.rs
+expression: parsed
+---
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "pool": "orchard",
+    "start_index": 0,
+    "subtrees": [
+      {
+        "root": "d4e323b3ae0cabfb6be4087fec8c66d9a9bbfc354bf1d9588b6620448182063b",
+        "end_height": 1707429
+      }
+    ]
+  },
+  "id": 123
+}

zebrad/tests/snapshots/z_getsubtreesbyindex_mainnet_orchard_338_1.snap

@@ -0,0 +1,18 @@
+---
+source: zebrad/tests/acceptance.rs
+expression: parsed
+---
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "pool": "orchard",
+    "start_index": 338,
+    "subtrees": [
+      {
+        "root": "52c8d1ea6ef49c6e0d6bb6fef4520e1e3851895a04b52bfac1b1cc0fbbb3f709",
+        "end_height": 1888929
+      }
+    ]
+  },
+  "id": 123
+}

zebrad/tests/snapshots/z_getsubtreesbyindex_mainnet_orchard_585_1.snap

@@ -0,0 +1,18 @@
+---
+source: zebrad/tests/acceptance.rs
+expression: parsed
+---
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "pool": "orchard",
+    "start_index": 585,
+    "subtrees": [
+      {
+        "root": "43c86869520ada7a2cd0deeecf3650f337f90bd4328b16144b0a278ad9fdaa08",
+        "end_height": 2000126
+      }
+    ]
+  },
+  "id": 123
+}

zebrad/tests/snapshots/z_getsubtreesbyindex_mainnet_sapling_0_1.snap

@@ -0,0 +1,18 @@
+---
+source: zebrad/tests/acceptance.rs
+expression: parsed
+---
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "pool": "sapling",
+    "start_index": 0,
+    "subtrees": [
+      {
+        "root": "754bb593ea42d231a7ddf367640f09bbf59dc00f2c1d2003cc340e0c016b5b13",
+        "end_height": 558822
+      }
+    ]
+  },
+  "id": 123
+}

zebrad/tests/snapshots/z_getsubtreesbyindex_mainnet_sapling_0_11.snap

@@ -0,0 +1,58 @@
+---
+source: zebrad/tests/acceptance.rs
+expression: parsed
+---
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "pool": "sapling",
+    "start_index": 0,
+    "subtrees": [
+      {
+        "root": "754bb593ea42d231a7ddf367640f09bbf59dc00f2c1d2003cc340e0c016b5b13",
+        "end_height": 558822
+      },
+      {
+        "root": "03654c3eacbb9b93e122cf6d77b606eae29610f4f38a477985368197fd68e02d",
+        "end_height": 670209
+      },
+      {
+        "root": "e2bf698f5ac10b44da560d11a5e1d5c191a82a968a2be0a6948aa8748b545160",
+        "end_height": 780364
+      },
+      {
+        "root": "e71be1fd3963a2d700ed374d01fdd4a70d8dda8189a8f6602e5fe66c2c66a11d",
+        "end_height": 839994
+      },
+      {
+        "root": "f1c57245fff8dbc2d3efe5a0953eafdedeb06e18a3ad4f1e4042ee76623f8032",
+        "end_height": 916404
+      },
+      {
+        "root": "ea8a275c49b47f4658fa4947bbac027bb1981e3009f292916cb1f7a93ac82c08",
+        "end_height": 985993
+      },
+      {
+        "root": "891b1e6bfec42e97c79ec505c7ae1b584cf47d4ed8f6cdfcad815b02a5496f67",
+        "end_height": 1056197
+      },
+      {
+        "root": "163838d11a0525898f410397ae6ec627fa18b3184a021e088806947b3c5ca718",
+        "end_height": 1109360
+      },
+      {
+        "root": "089a1f9d50a037cc66aba4400b1703bcbb66f5f2993fd0dd3bb726e359409167",
+        "end_height": 1169471
+      },
+      {
+        "root": "0fc8a6ad74e65764d9bc2fcd13866b994731dfda821e6618a1a308bb1eccf51a",
+        "end_height": 1256932
+      },
+      {
+        "root": "cdd92a1e884cf1914dae8345423203832ec7bbf9d95ae50b82e4327b39d6d912",
+        "end_height": 1363036
+      }
+    ]
+  },
+  "id": 123
+}

zebrad/tests/snapshots/z_getsubtreesbyindex_mainnet_sapling_1090_6.snap

@@ -0,0 +1,38 @@
+---
+source: zebrad/tests/acceptance.rs
+expression: parsed
+---
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "pool": "sapling",
+    "start_index": 1090,
+    "subtrees": [
+      {
+        "root": "507ebccc63e2ca45bd254975ca2b75f2a221a0167d514077472dc7ef1e04a638",
+        "end_height": 1857525
+      },
+      {
+        "root": "0240b333e243a38583b692b9ce5b2fbdfde41cfdd3f3067b5c7d94fda5387666",
+        "end_height": 1860908
+      },
+      {
+        "root": "bf299f8446bea806854e9b363ed890f83423cececd53293c81df92ff112aa43f",
+        "end_height": 1866377
+      },
+      {
+        "root": "4a4778e6242b3e708f040b4a373c8cdedd14c5c415ab20f32e46430507b1a346",
+        "end_height": 1889548
+      },
+      {
+        "root": "c1cffa22d637cb194102a1b02a57d2b47c67f54094f6773a38b2e53d3becb45f",
+        "end_height": 1997495
+      },
+      {
+        "root": "0ee7328354ad588ee9581b25a3e2b94dbb03db647b79538c8805d793c61b822d",
+        "end_height": 2056616
+      }
+    ]
+  },
+  "id": 123
+}

zebrad/tests/snapshots/z_getsubtreesbyindex_mainnet_sapling_17_1.snap

@@ -0,0 +1,18 @@
+---
+source: zebrad/tests/acceptance.rs
+expression: parsed
+---
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "pool": "sapling",
+    "start_index": 17,
+    "subtrees": [
+      {
+        "root": "89ac51fcf846eacb2e7bca32d9bf3793792d2d952c1fec11e52b56d7674c1c36",
+        "end_height": 1703171
+      }
+    ]
+  },
+  "id": 123
+}