From 59bdab17bf4f86b955446e4ab787394e26c4d9a1 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 30 Apr 2022 05:56:11 +1000 Subject: [PATCH] test(rpc): Add Rust tests for lightwalletd sync from Zebra (#4177) * Make the lightwalletd integration test take a test type * Configure lightwalletd tests based on the test type * Remove obsolete kill_on_error() in the lightwalletd test * Refactor to simplify the test function * Move LightwalletdTestType to the lightwalletd module * Create a test function that runs the full lightwalletd test suite * Actually use the cached Zebra state * Add checks for the new integration test modes * Populate the lightwalletd state dir in the FullSyncFromGenesis test * Fix up state handling, fail earlier if state is invalid * Adjust timeouts and regex escapes * Make state requirements for each test stricter * Move configs to the top of the test function * Allow unexpected lightwalletd cached state in some tests * Speed up tests slightly by removing an intermittent log check * Move timeout selection into test type methods * Move failure messages into test type methods * Turn a function argument into an enum field * Check lightwalletd state directly, rather than Zebra RPC results * Update gRPC tests for function argument changes * Remove duplicate env var constant and redundant code --- zebrad/tests/acceptance.rs | 394 +++++++++--------- zebrad/tests/common/failure_messages.rs | 121 ++++++ zebrad/tests/common/launch.rs | 20 +- zebrad/tests/common/lightwalletd.rs | 252 ++++++++++- .../tests/common/lightwalletd/wallet_grpc.rs | 40 +- zebrad/tests/common/mod.rs | 1 + 6 files changed, 580 insertions(+), 248 deletions(-) create mode 100644 zebrad/tests/common/failure_messages.rs diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index b9b45c0f..2bc8be23 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -35,12 +35,7 @@ use zebra_chain::{ use zebra_network::constants::PORT_IN_USE_ERROR; use zebra_state::constants::LOCK_FILE_ERROR; -use zebra_test::{ - args, - command::{ContextFrom, NO_MATCHES_REGEX_ITER}, - net::random_known_port, - prelude::*, -}; +use zebra_test::{args, command::ContextFrom, net::random_known_port, prelude::*}; mod common; @@ -49,11 +44,12 @@ use common::{ config::{default_test_config, persistent_test_config, testdir}, launch::{ spawn_zebrad_for_rpc_without_initial_peers, ZebradTestDirExt, BETWEEN_NODES_DELAY, - LAUNCH_DELAY, LIGHTWALLETD_DELAY, + LAUNCH_DELAY, }, lightwalletd::{ random_known_rpc_port_config, zebra_skip_lightwalletd_tests, LightWalletdTestDirExt, - LIGHTWALLETD_TEST_TIMEOUT, + LightwalletdTestType::{self, *}, + LIGHTWALLETD_DATA_DIR_VAR, }, sync::{ create_cached_database_height, sync_until, MempoolBehavior, LARGE_CHECKPOINT_TEST_HEIGHT, @@ -997,124 +993,7 @@ async fn rpc_endpoint() -> Result<()> { Ok(()) } -/// Failure log messages for any process, from the OS or shell. -/// -/// These messages show that the child process has failed. -/// So when we see them in the logs, we make the test fail. -const PROCESS_FAILURE_MESSAGES: &[&str] = &[ - // Linux - "Aborted", - // macOS / BSDs - "Abort trap", - // TODO: add other OS or C library errors? -]; - -/// Failure log messages from Zebra. -/// -/// These `zebrad` messages show that the `lightwalletd` integration test has failed. -/// So when we see them in the logs, we make the test fail. -const ZEBRA_FAILURE_MESSAGES: &[&str] = &[ - // Rust-specific panics - "The application panicked", - // RPC port errors - "Unable to start RPC server", - // TODO: disable if this actually happens during test zebrad shutdown - "Stopping RPC endpoint", - // Missing RPCs in zebrad logs (this log is from PR #3860) - // - // TODO: temporarily disable until enough RPCs are implemented, if needed - "Received unrecognized RPC request", - // RPC argument errors: parsing and data - // - // These logs are produced by jsonrpc_core inside Zebra, - // but it doesn't log them yet. - // - // TODO: log these errors in Zebra, and check for them in the Zebra logs? - "Invalid params", - "Method not found", -]; - -/// Failure log messages from lightwalletd. -/// -/// These `lightwalletd` messages show that the `lightwalletd` integration test has failed. -/// So when we see them in the logs, we make the test fail. -const LIGHTWALLETD_FAILURE_MESSAGES: &[&str] = &[ - // Go-specific panics - "panic:", - // Missing RPCs in lightwalletd logs - // TODO: temporarily disable until enough RPCs are implemented, if needed - "unable to issue RPC call", - // RPC response errors: parsing and data - // - // jsonrpc_core error messages from Zebra, - // received by lightwalletd and written to its logs - "Invalid params", - "Method not found", - // Early termination - // - // TODO: temporarily disable until enough RPCs are implemented, if needed - "Lightwalletd died with a Fatal error", - // Go json package error messages: - "json: cannot unmarshal", - "into Go value of type", - // lightwalletd custom RPC error messages from: - // https://github.com/adityapk00/lightwalletd/blob/master/common/common.go - "block requested is newer than latest block", - "Cache add failed", - "error decoding", - "error marshaling", - "error parsing JSON", - "error reading JSON response", - "error with", - // We expect these errors when lightwalletd reaches the end of the zebrad cached state - // "error requesting block: 0: Block not found", - // "error zcashd getblock rpc", - "received overlong message", - "received unexpected height block", - "Reorg exceeded max", - "unable to issue RPC call", - // Missing fields for each specific RPC - // - // get_block_chain_info - // - // invalid sapling height - "Got sapling height 0", - // missing BIP70 chain name, should be "main" or "test" - " chain ", - // missing branchID, should be 8 hex digits - " branchID \"", - // get_block - // - // a block error other than "-8: Block not found" - "error requesting block", - // a missing block with an incorrect error code - "Block not found", - // - // TODO: complete this list for each RPC with fields, if that RPC generates logs - // get_info - doesn't generate logs - // get_raw_transaction - might not generate logs - // z_get_tree_state - // get_address_txids - // get_address_balance - // get_address_utxos -]; - -/// Ignored failure logs for lightwalletd. -/// These regexes override the [`LIGHTWALLETD_FAILURE_MESSAGES`]. -/// -/// These `lightwalletd` messages look like failure messages, but they are actually ok. -/// So when we see them in the logs, we make the test continue. -const LIGHTWALLETD_IGNORE_MESSAGES: &[&str] = &[ - // Exceptions to lightwalletd custom RPC error messages: - // - // This log matches the "error with" RPC error message, - // but we expect Zebra to start with an empty state. - // - // TODO: this exception should not be used for the cached state tests (#3511) - r#"No Chain tip available yet","level":"warning","msg":"error with getblockchaininfo rpc, retrying"#, -]; - -/// Launch `zebrad` with an RPC port, and make sure `lightwalletd` works with Zebra. +/// Make sure `lightwalletd` works with Zebra, when both their states are empty. /// /// This test only runs when the `ZEBRA_TEST_LIGHTWALLETD` env var is set. /// @@ -1122,6 +1001,72 @@ const LIGHTWALLETD_IGNORE_MESSAGES: &[&str] = &[ #[test] #[cfg(not(target_os = "windows"))] fn lightwalletd_integration() -> Result<()> { + lightwalletd_integration_test(LaunchWithEmptyState) +} + +/// Make sure `lightwalletd` can sync from Zebra, in update sync mode. +/// +/// If is set, runs a quick sync, then a full sync. +/// If `LIGHTWALLETD_DATA_DIR` is not set, just runs a full sync. +/// +/// This test only runs when the `ZEBRA_TEST_LIGHTWALLETD`, +/// `ZEBRA_CACHED_STATE_DIR`, and `LIGHTWALLETD_DATA_DIR` env vars are set. +/// +/// This test doesn't work on Windows, so it is always skipped on that platform. +#[test] +#[cfg(not(target_os = "windows"))] +fn lightwalletd_update_sync() -> Result<()> { + lightwalletd_integration_test(UpdateCachedState) +} + +/// Make sure `lightwalletd` can fully sync from genesis using Zebra. +/// +/// This test only runs when the `ZEBRA_TEST_LIGHTWALLETD` and +/// `ZEBRA_CACHED_STATE_DIR` env vars are set. +/// +/// This test doesn't work on Windows, so it is always skipped on that platform. +#[test] +#[ignore] +#[cfg(not(target_os = "windows"))] +fn lightwalletd_full_sync() -> Result<()> { + lightwalletd_integration_test(FullSyncFromGenesis { + allow_lightwalletd_cached_state: false, + }) +} + +/// Make sure `lightwalletd` can sync from Zebra, in all available modes. +/// +/// Runs the tests in this order: +/// - launch lightwalletd with empty states, +/// - if `ZEBRA_CACHED_STATE_DIR` and `LIGHTWALLETD_DATA_DIR` are set: run a quick update sync, +/// - if `ZEBRA_CACHED_STATE_DIR` is set: run a full sync. +/// +/// These tests don't work on Windows, so they are always skipped on that platform. +#[test] +#[ignore] +#[cfg(not(target_os = "windows"))] +fn lightwalletd_test_suite() -> Result<()> { + lightwalletd_integration_test(LaunchWithEmptyState)?; + + // Only runs when ZEBRA_CACHED_STATE_DIR is set. + // When manually running the test suite, allow cached state in the full sync test. + lightwalletd_integration_test(FullSyncFromGenesis { + allow_lightwalletd_cached_state: true, + })?; + + // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set + lightwalletd_integration_test(UpdateCachedState)?; + + Ok(()) +} + +/// Run a lightwalletd integration test with a configuration for `test_type`. +/// +/// Set `allow_cached_state_for_full_sync` to speed up manual full sync tests. +/// +/// The random ports in this test can cause [rare port conflicts.](#Note on port conflict) +#[cfg(not(target_os = "windows"))] +fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> { zebra_test::init(); // Skip the test unless the user specifically asked for it @@ -1129,29 +1074,62 @@ fn lightwalletd_integration() -> Result<()> { return Ok(()); } - // Launch zebrad + // Get the zebrad and lightwalletd configs - // Write a configuration that has RPC listen_addr set - // [Note on port conflict](#Note on port conflict) - let mut config = random_known_rpc_port_config()?; + // Handle the Zebra state directory based on the test type: + // - LaunchWithEmptyState: ignore the state directory + // - FullSyncFromGenesis & UpdateCachedState: + // skip the test if it is not available, timeout if it is not populated - let zdir = testdir()?.with_config(&mut config)?; - let mut zebrad = zdir - .spawn_child(args!["start"])? - .with_timeout(LAUNCH_DELAY) - .with_failure_regex_iter( - // TODO: replace with a function that returns the full list and correct return type - ZEBRA_FAILURE_MESSAGES - .iter() - .chain(PROCESS_FAILURE_MESSAGES) - .cloned(), - NO_MATCHES_REGEX_ITER.iter().cloned(), + // Write a configuration that has RPC listen_addr set. + // If the state path env var is set, use it in the config. + let config = if let Some(config) = test_type.zebrad_config() { + config? + } else { + return Ok(()); + }; + + // Handle the lightwalletd state directory based on the test type: + // - LaunchWithEmptyState: ignore the state directory + // - FullSyncFromGenesis: use it if available, timeout if it is already populated + // - UpdateCachedState: skip the test if it is not available, timeout if it is not populated + let lightwalletd_state_path = test_type.lightwalletd_state_path(); + + if test_type.needs_lightwalletd_cached_state() && lightwalletd_state_path.is_none() { + tracing::info!( + "skipped {test_type:?} lightwalletd test, \ + set the {LIGHTWALLETD_DATA_DIR_VAR:?} environment variable to run the test", ); + return Ok(()); + } + + tracing::info!(?test_type, "running lightwalletd & zebrad integration test"); + + // Get the lists of process failure logs + let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); + + let (lightwalletd_failure_messages, lightwalletd_ignore_messages) = + test_type.lightwalletd_failure_messages(); + + // Launch zebrad + let zdir = testdir()?.with_exact_config(&config)?; + let mut zebrad = zdir + .spawn_child(args!["start"])? + .with_timeout(test_type.zebrad_timeout()) + .with_failure_regex_iter(zebrad_failure_messages, zebrad_ignore_messages); + + if test_type.needs_zebra_cached_state() { + zebrad.expect_stdout_line_matches(r"loaded Zebra state cache tip=.*Height\([0-9]{7}\)")?; + } else { + // Timeout the test if we're somehow accidentally using a cached state + zebrad.expect_stdout_line_matches("loaded Zebra state cache tip=None")?; + } + // Wait until `zebrad` has opened the RPC endpoint - zebrad.expect_stdout_line_matches( + zebrad.expect_stdout_line_matches(regex::escape( format!("Opened RPC endpoint at {}", config.rpc.listen_addr.unwrap()).as_str(), - )?; + ))?; // Launch lightwalletd @@ -1160,37 +1138,40 @@ fn lightwalletd_integration() -> Result<()> { let ldir = ldir.with_lightwalletd_config(config.rpc.listen_addr.unwrap())?; // Launch the lightwalletd process - let result = ldir.spawn_lightwalletd_child(args![]); - let (lightwalletd, zebrad) = zebrad.kill_on_error(result)?; + let lightwalletd = if test_type == LaunchWithEmptyState { + ldir.spawn_lightwalletd_child(None, args![])? + } else { + ldir.spawn_lightwalletd_child(lightwalletd_state_path, args![])? + }; + let mut lightwalletd = lightwalletd - .with_timeout(LIGHTWALLETD_DELAY) - .with_failure_regex_iter( - // TODO: replace with a function that returns the full list and correct return type - LIGHTWALLETD_FAILURE_MESSAGES - .iter() - .chain(PROCESS_FAILURE_MESSAGES) - .cloned(), - // TODO: some exceptions do not apply to the cached state tests (#3511) - LIGHTWALLETD_IGNORE_MESSAGES.iter().cloned(), - ); + .with_timeout(test_type.lightwalletd_timeout()) + .with_failure_regex_iter(lightwalletd_failure_messages, lightwalletd_ignore_messages); // Wait until `lightwalletd` has launched - let result = lightwalletd.expect_stdout_line_matches("Starting gRPC server"); - let (_, zebrad) = zebrad.kill_on_error(result)?; + lightwalletd.expect_stdout_line_matches(regex::escape("Starting gRPC server"))?; // Check that `lightwalletd` is calling the expected Zebra RPCs // getblockchaininfo - // - // TODO: update branchID when we're using cached state (#3511) - // add "Waiting for zcashd height to reach Sapling activation height" - let result = lightwalletd.expect_stdout_line_matches( - "Got sapling height 419200 block height [0-9]+ chain main branchID 00000000", - ); - let (_, zebrad) = zebrad.kill_on_error(result)?; + if test_type.needs_zebra_cached_state() { + lightwalletd.expect_stdout_line_matches( + "Got sapling height 419200 block height [0-9]{7} chain main branchID e9ff75a6", + )?; + } else { + // Timeout the test if we're somehow accidentally using a cached state in our temp dir + lightwalletd.expect_stdout_line_matches( + "Got sapling height 419200 block height [0-9]{1,6} chain main branchID 00000000", + )?; + } - let result = lightwalletd.expect_stdout_line_matches("Found 0 blocks in cache"); - let (_, zebrad) = zebrad.kill_on_error(result)?; + if test_type.needs_lightwalletd_cached_state() { + // TODO: expect `[0-9]{7}` when we're using the tip cached state (#4155) + lightwalletd.expect_stdout_line_matches("Found [0-9]{6,7} blocks in cache")?; + } else if !test_type.allow_lightwalletd_cached_state() { + // Timeout the test if we're somehow accidentally using a cached state in our temp dir + lightwalletd.expect_stdout_line_matches("Found 0 blocks in cache")?; + } // getblock with the first Sapling block in Zebra's state // @@ -1199,34 +1180,44 @@ fn lightwalletd_integration() -> Result<()> { // // The log also depends on what is in Zebra's state: // + // # Cached Zebra State + // + // lightwalletd ingests blocks into its cache. + // // # Empty Zebra State // // lightwalletd tries to download the Sapling activation block, but it's not in the state. // - // Until the Sapling activation block has been downloaded, lightwalletd will log Zebra's RPC error: - // "error requesting block: 0: Block not found" - // We also get a similar log when lightwalletd reaches the end of Zebra's cache. - // - // # Cached Zebra State - // - // After the first successful getblock call, lightwalletd will log: - // "Block hash changed, clearing mempool clients" - // But we can't check for that, because it can come before or after the Ingestor log. - // - // TODO: expect Ingestor log when we're using cached state (#3511) - // "Ingestor adding block to cache" - let result = lightwalletd.expect_stdout_line_matches(regex::escape( - "Waiting for zcashd height to reach Sapling activation height (419200)", - )); - let (_, zebrad) = zebrad.kill_on_error(result)?; + // Until the Sapling activation block has been downloaded, + // lightwalletd will keep retrying getblock. + if test_type.needs_zebra_cached_state() { + lightwalletd.expect_stdout_line_matches(regex::escape("Ingestor adding block to cache"))?; + } else { + lightwalletd.expect_stdout_line_matches(regex::escape( + "Waiting for zcashd height to reach Sapling activation height (419200)", + ))?; + } - // (next RPC) - // - // TODO: add extra checks when we add new Zebra RPCs + if matches!(test_type, UpdateCachedState | FullSyncFromGenesis { .. }) { + // Wait for Zebra to sync its cached state to the chain tip + zebrad.expect_stdout_line_matches(regex::escape("sync_percent=100"))?; + + // Wait for lightwalletd to sync to Zebra's tip + lightwalletd.expect_stdout_line_matches(regex::escape("Ingestor waiting for block"))?; + + // Check Zebra is still at the tip (also clears and prints Zebra's logs) + zebrad.expect_stdout_line_matches(regex::escape("sync_percent=100"))?; + + // lightwalletd doesn't log anything when we've reached the tip. + // But when it gets near the tip, it starts using the mempool. + lightwalletd.expect_stdout_line_matches(regex::escape( + "Block hash changed, clearing mempool clients", + ))?; + lightwalletd.expect_stdout_line_matches(regex::escape("Adding new mempool txid"))?; + } // Cleanup both processes - let result = lightwalletd.kill(); - let (_, mut zebrad) = zebrad.kill_on_error(result)?; + lightwalletd.kill()?; zebrad.kill()?; let lightwalletd_output = lightwalletd.wait_with_output()?.assert_failure()?; @@ -1482,26 +1473,27 @@ where async fn fully_synced_rpc_test() -> Result<()> { zebra_test::init(); - // TODO: reuse code from https://github.com/ZcashFoundation/zebra/pull/4177/ - // to get the cached_state_path - const CACHED_STATE_PATH_VAR: &str = "ZEBRA_CACHED_STATE_PATH"; - let cached_state_path = match env::var_os(CACHED_STATE_PATH_VAR) { - Some(argument) => PathBuf::from(argument), - None => { - tracing::info!( - "skipped send transactions using lightwalletd test, \ - set the {CACHED_STATE_PATH_VAR:?} environment variable to run the test", - ); - return Ok(()); - } + // We're only using cached Zebra state here, so this test type is the most similar + let test_type = LightwalletdTestType::FullSyncFromGenesis { + allow_lightwalletd_cached_state: false, }; + // Handle the Zebra state directory + let cached_state_path = test_type.zebrad_state_path(); + + if cached_state_path.is_none() { + tracing::info!("skipping fully synced zebrad RPC test"); + return Ok(()); + }; + + tracing::info!("running fully synced zebrad RPC test"); + let network = Network::Mainnet; let (_zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers( network, - cached_state_path, - LIGHTWALLETD_TEST_TIMEOUT, + cached_state_path.unwrap(), + test_type.zebrad_timeout(), )?; // Make a getblock test that works only on synced node (high block number). diff --git a/zebrad/tests/common/failure_messages.rs b/zebrad/tests/common/failure_messages.rs new file mode 100644 index 00000000..0068b0e0 --- /dev/null +++ b/zebrad/tests/common/failure_messages.rs @@ -0,0 +1,121 @@ +//! Failure messages logged by test child processes. +//! +//! # Warning +//! +//! Test functions in this file will not be run. +//! This file is only for test library code. + +/// Failure log messages for any process, from the OS or shell. +/// +/// These messages show that the child process has failed. +/// So when we see them in the logs, we make the test fail. +pub const PROCESS_FAILURE_MESSAGES: &[&str] = &[ + // Linux + "Aborted", + // macOS / BSDs + "Abort trap", + // TODO: add other OS or C library errors? +]; + +/// Failure log messages from Zebra. +/// +/// These `zebrad` messages show that the `lightwalletd` integration test has failed. +/// So when we see them in the logs, we make the test fail. +pub const ZEBRA_FAILURE_MESSAGES: &[&str] = &[ + // Rust-specific panics + "The application panicked", + // RPC port errors + "Unable to start RPC server", + // TODO: disable if this actually happens during test zebrad shutdown + "Stopping RPC endpoint", + // Missing RPCs in zebrad logs (this log is from PR #3860) + // + // TODO: temporarily disable until enough RPCs are implemented, if needed + "Received unrecognized RPC request", + // RPC argument errors: parsing and data + // + // These logs are produced by jsonrpc_core inside Zebra, + // but it doesn't log them yet. + // + // TODO: log these errors in Zebra, and check for them in the Zebra logs? + "Invalid params", + "Method not found", +]; + +/// Failure log messages from lightwalletd. +/// +/// These `lightwalletd` messages show that the `lightwalletd` integration test has failed. +/// So when we see them in the logs, we make the test fail. +pub const LIGHTWALLETD_FAILURE_MESSAGES: &[&str] = &[ + // Go-specific panics + "panic:", + // Missing RPCs in lightwalletd logs + // TODO: temporarily disable until enough RPCs are implemented, if needed + "unable to issue RPC call", + // RPC response errors: parsing and data + // + // jsonrpc_core error messages from Zebra, + // received by lightwalletd and written to its logs + "Invalid params", + "Method not found", + // Early termination + // + // TODO: temporarily disable until enough RPCs are implemented, if needed + "Lightwalletd died with a Fatal error", + // Go json package error messages: + "json: cannot unmarshal", + "into Go value of type", + // lightwalletd custom RPC error messages from: + // https://github.com/adityapk00/lightwalletd/blob/master/common/common.go + "block requested is newer than latest block", + "Cache add failed", + "error decoding", + "error marshaling", + "error parsing JSON", + "error reading JSON response", + "error with", + // We expect these errors when lightwalletd reaches the end of the zebrad cached state + // "error requesting block: 0: Block not found", + // "error zcashd getblock rpc", + "received overlong message", + "received unexpected height block", + "Reorg exceeded max", + "unable to issue RPC call", + // Missing fields for each specific RPC + // + // get_block_chain_info + // + // invalid sapling height + "Got sapling height 0", + // missing BIP70 chain name, should be "main" or "test" + " chain ", + // missing branchID, should be 8 hex digits + " branchID \"", + // get_block + // + // a block error other than "-8: Block not found" + "error requesting block", + // a missing block with an incorrect error code + "Block not found", + // + // TODO: complete this list for each RPC with fields, if that RPC generates logs + // get_info - doesn't generate logs + // get_raw_transaction - might not generate logs + // z_get_tree_state + // get_address_txids + // get_address_balance + // get_address_utxos +]; + +/// Ignored failure logs for lightwalletd. +/// These regexes override the [`LIGHTWALLETD_FAILURE_MESSAGES`]. +/// +/// These `lightwalletd` messages look like failure messages, but they are actually ok. +/// So when we see them in the logs, we make the test continue. +pub const LIGHTWALLETD_EMPTY_ZEBRA_STATE_IGNORE_MESSAGES: &[&str] = &[ + // Exceptions to lightwalletd custom RPC error messages: + // + // This log matches the "error with" RPC error message, + // but we expect Zebra to start with an empty state. + r#"No Chain tip available yet","level":"warning","msg":"error with getblockchaininfo rpc, retrying"#, +]; diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 6701641c..1221b6c5 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -23,9 +23,9 @@ use zebra_test::{ }; use zebrad::config::ZebradConfig; -use crate::{ - common::lightwalletd::random_known_rpc_port_config, PROCESS_FAILURE_MESSAGES, - ZEBRA_FAILURE_MESSAGES, +use crate::common::{ + failure_messages::{PROCESS_FAILURE_MESSAGES, ZEBRA_FAILURE_MESSAGES}, + lightwalletd::random_known_rpc_port_config, }; /// After we launch `zebrad`, wait this long for the command to start up, @@ -42,10 +42,20 @@ pub const LAUNCH_DELAY: Duration = Duration::from_secs(15); /// it is using for its RPCs. pub const LIGHTWALLETD_DELAY: Duration = Duration::from_secs(60); -/// The amount of time we wait between launching two -/// conflicting nodes. +/// The amount of time we wait between launching two conflicting nodes. pub const BETWEEN_NODES_DELAY: Duration = Duration::from_secs(2); +/// The amount of time we wait for lightwalletd to update to the tip. +/// +/// The cached tip can be a few days old, and Zebra needs time to activate its mempool. +pub const LIGHTWALLETD_UPDATE_TIP_DELAY: Duration = Duration::from_secs(10 * 60); + +/// The amount of time we wait for lightwalletd to do a full sync to the tip. +/// +/// `lightwalletd` takes about half an hour to fully sync, +/// and Zebra needs time to activate its mempool. +pub const LIGHTWALLETD_FULL_SYNC_TIP_DELAY: Duration = Duration::from_secs(60 * 60); + /// Extension trait for methods on `tempfile::TempDir` for using it as a test /// directory for `zebrad`. pub trait ZebradTestDirExt diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index a53bfd8e..4f50cda9 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -5,16 +5,34 @@ //! Test functions in this file will not be run. //! This file is only for test library code. -use std::{env, net::SocketAddr, path::Path, time::Duration}; +use std::{ + env, + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; use zebra_test::{ - command::{Arguments, TestChild, TestDirExt}, + command::{Arguments, TestChild, TestDirExt, NO_MATCHES_REGEX_ITER}, net::random_known_port, prelude::*, }; use zebrad::config::ZebradConfig; -use super::{config::default_test_config, launch::ZebradTestDirExt}; +use super::{ + cached_state::ZEBRA_CACHED_STATE_DIR_VAR, + config::default_test_config, + failure_messages::{ + LIGHTWALLETD_EMPTY_ZEBRA_STATE_IGNORE_MESSAGES, LIGHTWALLETD_FAILURE_MESSAGES, + PROCESS_FAILURE_MESSAGES, ZEBRA_FAILURE_MESSAGES, + }, + launch::{ + ZebradTestDirExt, LIGHTWALLETD_DELAY, LIGHTWALLETD_FULL_SYNC_TIP_DELAY, + LIGHTWALLETD_UPDATE_TIP_DELAY, + }, +}; + +use LightwalletdTestType::*; pub mod send_transaction_test; pub mod wallet_grpc; @@ -28,7 +46,16 @@ pub mod wallet_grpc; /// /// This environmental variable is used to enable the lightwalletd tests. /// But the network tests are *disabled* by their environmental variables. -const ZEBRA_TEST_LIGHTWALLETD: &str = "ZEBRA_TEST_LIGHTWALLETD"; +pub const ZEBRA_TEST_LIGHTWALLETD: &str = "ZEBRA_TEST_LIGHTWALLETD"; + +/// Optional environment variable with the cached state for lightwalletd. +/// +/// Required for [`LightwalletdTestType::UpdateCachedState`], +/// so we can test lightwalletd RPC integration with a populated state. +/// +/// Can also be used to speed up the [`sending_transactions_using_lightwalletd`] test, +/// by skipping the lightwalletd initial sync. +pub const LIGHTWALLETD_DATA_DIR_VAR: &str = "LIGHTWALLETD_DATA_DIR"; /// The maximum time that a `lightwalletd` integration test is expected to run. pub const LIGHTWALLETD_TEST_TIMEOUT: Duration = Duration::from_secs(60 * 60); @@ -75,16 +102,20 @@ pub trait LightWalletdTestDirExt: ZebradTestDirExt where Self: AsRef + Sized, { - /// Spawn `lightwalletd` with `args` as a child process in this test directory, - /// potentially taking ownership of the tempdir for the duration of the - /// child process. + /// Spawn `lightwalletd` with `lightwalletd_state_path`, and `extra_args`, + /// as a child process in this test directory, + /// potentially taking ownership of the tempdir for the duration of the child process. /// /// By default, launch a working test instance with logging, and avoid port conflicts. /// /// # Panics /// /// If there is no lightwalletd config in the test directory. - fn spawn_lightwalletd_child(self, extra_args: Arguments) -> Result>; + fn spawn_lightwalletd_child( + self, + lightwalletd_state_path: impl Into>, + extra_args: Arguments, + ) -> Result>; /// Create a config file and use it for all subsequently spawned `lightwalletd` processes. /// Returns an error if the config already exists. @@ -98,9 +129,13 @@ impl LightWalletdTestDirExt for T where Self: TestDirExt + AsRef + Sized, { - fn spawn_lightwalletd_child(self, extra_args: Arguments) -> Result> { - let dir = self.as_ref().to_owned(); - let default_config_path = dir.join("lightwalletd-zcash.conf"); + fn spawn_lightwalletd_child( + self, + lightwalletd_state_path: impl Into>, + extra_args: Arguments, + ) -> Result> { + let test_dir = self.as_ref().to_owned(); + let default_config_path = test_dir.join("lightwalletd-zcash.conf"); assert!( default_config_path.exists(), @@ -119,9 +154,24 @@ where args.set_parameter("--zcash-conf-path", zcash_conf_path); // the lightwalletd cache directory - // - // TODO: create a sub-directory for lightwalletd - args.set_parameter("--data-dir", dir.to_str().expect("Path is valid Unicode")); + if let Some(lightwalletd_state_path) = lightwalletd_state_path.into() { + args.set_parameter( + "--data-dir", + lightwalletd_state_path + .to_str() + .expect("path is valid Unicode"), + ); + } else { + let empty_state_path = test_dir.join("lightwalletd_state"); + + std::fs::create_dir(&empty_state_path) + .expect("unexpected failure creating lightwalletd state sub-directory"); + + args.set_parameter( + "--data-dir", + empty_state_path.to_str().expect("path is valid Unicode"), + ); + } // log to standard output // @@ -163,3 +213,177 @@ where Ok(self) } } + +/// The type of lightwalletd integration test that we're running. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum LightwalletdTestType { + /// Launch with an empty Zebra and lightwalletd state. + LaunchWithEmptyState, + + /// Do a full sync from an empty lightwalletd state. + /// + /// This test requires a cached Zebra state. + FullSyncFromGenesis { + /// Should the test allow a cached lightwalletd state? + /// + /// If `false`, the test fails if the lightwalletd state is populated. + allow_lightwalletd_cached_state: bool, + }, + + /// Sync to tip from a lightwalletd cached state. + /// + /// This test requires a cached Zebra and lightwalletd state. + UpdateCachedState, +} + +impl LightwalletdTestType { + /// Does this test need a Zebra cached state? + pub fn needs_zebra_cached_state(&self) -> bool { + match self { + LaunchWithEmptyState => false, + FullSyncFromGenesis { .. } | UpdateCachedState => true, + } + } + + /// Does this test need a lightwalletd cached state? + pub fn needs_lightwalletd_cached_state(&self) -> bool { + match self { + LaunchWithEmptyState | FullSyncFromGenesis { .. } => false, + UpdateCachedState => true, + } + } + + /// Does this test allow a lightwalletd cached state, even if it is not required? + pub fn allow_lightwalletd_cached_state(&self) -> bool { + match self { + LaunchWithEmptyState => false, + FullSyncFromGenesis { + allow_lightwalletd_cached_state, + } => *allow_lightwalletd_cached_state, + UpdateCachedState => true, + } + } + + /// Returns the Zebra state path for this test, if set. + pub fn zebrad_state_path(&self) -> Option { + match env::var_os(ZEBRA_CACHED_STATE_DIR_VAR) { + Some(path) => Some(path.into()), + None => { + tracing::info!( + "skipped {self:?} lightwalletd test, \ + set the {ZEBRA_CACHED_STATE_DIR_VAR:?} environment variable to run the test", + ); + + None + } + } + } + + /// Returns a Zebra config for this test. + /// + /// Returns `None` if the test should be skipped, + /// and `Some(Err(_))` if the config could not be created. + pub fn zebrad_config(&self) -> Option> { + if !self.needs_zebra_cached_state() { + return Some(random_known_rpc_port_config()); + } + + let zebra_state_path = self.zebrad_state_path()?; + + let mut config = match random_known_rpc_port_config() { + Ok(config) => config, + Err(error) => return Some(Err(error)), + }; + + config.sync.lookahead_limit = zebrad::components::sync::DEFAULT_LOOKAHEAD_LIMIT; + + config.state.ephemeral = false; + config.state.cache_dir = zebra_state_path; + + Some(Ok(config)) + } + + /// Returns the lightwalletd state path for this test, if set. + pub fn lightwalletd_state_path(&self) -> Option { + env::var_os(LIGHTWALLETD_DATA_DIR_VAR).map(Into::into) + } + + /// Returns the `zebrad` timeout for this test type. + pub fn zebrad_timeout(&self) -> Duration { + match self { + LaunchWithEmptyState => LIGHTWALLETD_DELAY, + FullSyncFromGenesis { .. } | UpdateCachedState => LIGHTWALLETD_UPDATE_TIP_DELAY, + } + } + + /// Returns the `lightwalletd` timeout for this test type. + pub fn lightwalletd_timeout(&self) -> Duration { + match self { + LaunchWithEmptyState => LIGHTWALLETD_DELAY, + UpdateCachedState => LIGHTWALLETD_UPDATE_TIP_DELAY, + FullSyncFromGenesis { .. } => LIGHTWALLETD_FULL_SYNC_TIP_DELAY, + } + } + + /// Returns Zebra log regexes that indicate the tests have failed, + /// and regexes of any failures that should be ignored. + pub fn zebrad_failure_messages(&self) -> (Vec, Vec) { + let mut zebrad_failure_messages: Vec = ZEBRA_FAILURE_MESSAGES + .iter() + .chain(PROCESS_FAILURE_MESSAGES) + .map(ToString::to_string) + .collect(); + + if self.needs_zebra_cached_state() { + // Fail if we need a cached Zebra state, but it's empty + zebrad_failure_messages.push("loaded Zebra state cache tip=None".to_string()); + } + if *self == LaunchWithEmptyState { + // Fail if we need an empty Zebra state, but it has blocks + zebrad_failure_messages + .push(r"loaded Zebra state cache tip=.*Height\([1-9][0-9]*\)".to_string()); + } + + let zebrad_ignore_messages = Vec::new(); + + (zebrad_failure_messages, zebrad_ignore_messages) + } + + /// Returns `lightwalletd` log regexes that indicate the tests have failed, + /// and regexes of any failures that should be ignored. + pub fn lightwalletd_failure_messages(&self) -> (Vec, Vec) { + let mut lightwalletd_failure_messages: Vec = LIGHTWALLETD_FAILURE_MESSAGES + .iter() + .chain(PROCESS_FAILURE_MESSAGES) + .map(ToString::to_string) + .collect(); + + // Zebra state failures + if self.needs_zebra_cached_state() { + // Fail if we need a cached Zebra state, but it's empty + lightwalletd_failure_messages.push("No Chain tip available yet".to_string()); + } + + // lightwalletd state failures + if self.needs_lightwalletd_cached_state() { + // Fail if we need a cached lightwalletd state, but it isn't near the tip + // + // TODO: fail on `[0-9]{1,6}` when we're using the tip cached state (#4155) + lightwalletd_failure_messages.push("Found [0-9]{1,5} blocks in cache".to_string()); + } + if !self.allow_lightwalletd_cached_state() { + // Fail if we need an empty lightwalletd state, but it has blocks + lightwalletd_failure_messages.push("Found [1-9][0-9]* blocks in cache".to_string()); + } + + let lightwalletd_ignore_messages = if *self == LaunchWithEmptyState { + LIGHTWALLETD_EMPTY_ZEBRA_STATE_IGNORE_MESSAGES.iter() + } else { + NO_MATCHES_REGEX_ITER.iter() + } + .map(ToString::to_string) + .collect(); + + (lightwalletd_failure_messages, lightwalletd_ignore_messages) + } +} diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index 1c9b6c9b..d316cac5 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -6,22 +6,12 @@ use tempfile::TempDir; use zebra_test::{args, net::random_known_port, prelude::*}; -use crate::{ - common::{ - config::testdir, - lightwalletd::{LightWalletdTestDirExt, LIGHTWALLETD_TEST_TIMEOUT}, - }, - LIGHTWALLETD_FAILURE_MESSAGES, LIGHTWALLETD_IGNORE_MESSAGES, PROCESS_FAILURE_MESSAGES, -}; +use crate::common::{config::testdir, lightwalletd::LightWalletdTestDirExt}; + +use super::LightwalletdTestType; tonic::include_proto!("cash.z.wallet.sdk.rpc"); -/// Optional environment variable with the cached state for lightwalletd. -/// -/// Can be used to speed up the [`sending_transactions_using_lightwalletd`] test, by allowing the -/// test to reuse the cached lightwalletd synchronization data. -const LIGHTWALLETD_DATA_DIR_VAR: &str = "LIGHTWALLETD_DATA_DIR"; - /// Type alias for the RPC client to communicate with a lightwalletd instance. pub type LightwalletdRpcClient = compact_tx_streamer_client::CompactTxStreamerClient; @@ -32,29 +22,23 @@ pub type LightwalletdRpcClient = pub fn spawn_lightwalletd_with_rpc_server( zebrad_rpc_address: SocketAddr, ) -> Result<(TestChild, u16)> { + // We're using cached Zebra state here, so this test type is the most similar + let test_type = LightwalletdTestType::UpdateCachedState; + let lightwalletd_dir = testdir()?.with_lightwalletd_config(zebrad_rpc_address)?; let lightwalletd_rpc_port = random_known_port(); let lightwalletd_rpc_address = format!("127.0.0.1:{lightwalletd_rpc_port}"); - let mut arguments = args!["--grpc-bind-addr": lightwalletd_rpc_address]; + let arguments = args!["--grpc-bind-addr": lightwalletd_rpc_address]; - if let Ok(data_dir) = env::var(LIGHTWALLETD_DATA_DIR_VAR) { - arguments.set_parameter("--data-dir", data_dir); - } + let (lightwalletd_failure_messages, lightwalletd_ignore_messages) = + test_type.lightwalletd_failure_messages(); let mut lightwalletd = lightwalletd_dir - .spawn_lightwalletd_child(arguments)? - .with_timeout(LIGHTWALLETD_TEST_TIMEOUT) - .with_failure_regex_iter( - // TODO: replace with a function that returns the full list and correct return type - LIGHTWALLETD_FAILURE_MESSAGES - .iter() - .chain(PROCESS_FAILURE_MESSAGES) - .cloned(), - // TODO: some exceptions do not apply to the cached state tests (#3511) - LIGHTWALLETD_IGNORE_MESSAGES.iter().cloned(), - ); + .spawn_lightwalletd_child(test_type.lightwalletd_state_path(), arguments)? + .with_timeout(test_type.lightwalletd_timeout()) + .with_failure_regex_iter(lightwalletd_failure_messages, lightwalletd_ignore_messages); lightwalletd.expect_stdout_line_matches("Starting gRPC server")?; lightwalletd.expect_stdout_line_matches("Waiting for block")?; diff --git a/zebrad/tests/common/mod.rs b/zebrad/tests/common/mod.rs index 70847ac3..988b5c62 100644 --- a/zebrad/tests/common/mod.rs +++ b/zebrad/tests/common/mod.rs @@ -12,6 +12,7 @@ pub mod cached_state; pub mod check; pub mod config; +pub mod failure_messages; pub mod launch; pub mod lightwalletd; pub mod sync;