From f687ab947ff3a745c0d507d1ae76a37cc3f4a839 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 25 Mar 2022 09:25:31 -0300 Subject: [PATCH] feat(rpc): Implement `getblockchaininfo` RPC method (#3891) * Implement `getblockchaininfo` RPC method * add a test for `get_blockchain_info` * fix tohex/fromhex * move comment * Update lightwalletd acceptance test for getblockchaininfo RPC (#3914) * change(rpc): Return getblockchaininfo network upgrades in height order (#3915) * Update lightwalletd acceptance test for getblockchaininfo RPC * Update some doc comments for network upgrades * List network upgrades in order in the getblockchaininfo RPC Also: - Use a constant for the "missing consensus branch ID" RPC value - Simplify fetching consensus branch IDs - Make RPC type derives consistent - Update RPC type documentation * Make RPC type derives consistent * Fix a confusing test comment * get hashand height at the same time * fix estimated_height * fix lint * add extra check Co-authored-by: Janito Vaqueiro Ferreira Filho * fix typo Co-authored-by: Janito Vaqueiro Ferreira Filho * split test Co-authored-by: Janito Vaqueiro Ferreira Filho * fix(rpc): ignore an expected error in the RPC acceptance tests (#3961) * Add ignored regexes to test command failure regex methods * Ignore empty chain error in getblockchaininfo We expect this error when zebrad starts up with an empty state. Co-authored-by: teor Co-authored-by: Janito Vaqueiro Ferreira Filho Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- Cargo.lock | 4 + zebra-chain/src/chain_tip.rs | 7 + zebra-chain/src/chain_tip/mock.rs | 25 ++- zebra-chain/src/parameters/network.rs | 9 + zebra-chain/src/parameters/network_upgrade.rs | 67 ++++++- zebra-chain/src/parameters/tests.rs | 18 ++ zebra-rpc/Cargo.toml | 4 + zebra-rpc/src/methods.rs | 161 +++++++++++++++-- zebra-rpc/src/methods/tests/prop.rs | 98 ++++++++++- zebra-state/src/service/chain_tip.rs | 5 + zebra-test/src/command.rs | 166 ++++++++++++++---- zebra-test/tests/command.rs | 99 ++++++++++- zebrad/Cargo.toml | 3 +- zebrad/tests/acceptance.rs | 63 ++++++- 14 files changed, 650 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e61d158..d4fdc30f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1942,6 +1942,7 @@ checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" dependencies = [ "autocfg 1.1.0", "hashbrown", + "serde", ] [[package]] @@ -3870,6 +3871,7 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" dependencies = [ + "indexmap", "itoa 1.0.1", "ryu", "serde", @@ -5746,9 +5748,11 @@ dependencies = [ name = "zebra-rpc" version = "1.0.0-beta.0" dependencies = [ + "chrono", "futures", "hex", "hyper", + "indexmap", "jsonrpc-core", "jsonrpc-derive", "jsonrpc-http-server", diff --git a/zebra-chain/src/chain_tip.rs b/zebra-chain/src/chain_tip.rs index 61c6b684..923302ac 100644 --- a/zebra-chain/src/chain_tip.rs +++ b/zebra-chain/src/chain_tip.rs @@ -25,6 +25,9 @@ pub trait ChainTip { /// Return the block hash of the best chain tip. fn best_tip_hash(&self) -> Option; + /// Return the height and the hash of the best chain tip. + fn best_tip_height_and_hash(&self) -> Option<(block::Height, block::Hash)>; + /// Return the block time of the best chain tip. fn best_tip_block_time(&self) -> Option>; @@ -70,6 +73,10 @@ impl ChainTip for NoChainTip { None } + fn best_tip_height_and_hash(&self) -> Option<(block::Height, block::Hash)> { + None + } + fn best_tip_block_time(&self) -> Option> { None } diff --git a/zebra-chain/src/chain_tip/mock.rs b/zebra-chain/src/chain_tip/mock.rs index 839a39a5..b13231d6 100644 --- a/zebra-chain/src/chain_tip/mock.rs +++ b/zebra-chain/src/chain_tip/mock.rs @@ -12,6 +12,9 @@ pub struct MockChainTipSender { /// A sender that sets the `best_tip_height` of a [`MockChainTip`]. best_tip_height: watch::Sender>, + /// A sender that sets the `best_tip_hash` of a [`MockChainTip`]. + best_tip_hash: watch::Sender>, + /// A sender that sets the `best_tip_block_time` of a [`MockChainTip`]. best_tip_block_time: watch::Sender>>, } @@ -22,6 +25,9 @@ pub struct MockChainTip { /// A mocked `best_tip_height` value set by the [`MockChainTipSender`]. best_tip_height: watch::Receiver>, + /// A mocked `best_tip_hash` value set by the [`MockChainTipSender`]. + best_tip_hash: watch::Receiver>, + /// A mocked `best_tip_height` value set by the [`MockChainTipSender`]. best_tip_block_time: watch::Receiver>>, } @@ -35,15 +41,18 @@ impl MockChainTip { /// Initially, the best tip height is [`None`]. pub fn new() -> (Self, MockChainTipSender) { let (height_sender, height_receiver) = watch::channel(None); + let (hash_sender, hash_receiver) = watch::channel(None); let (time_sender, time_receiver) = watch::channel(None); let mock_chain_tip = MockChainTip { best_tip_height: height_receiver, + best_tip_hash: hash_receiver, best_tip_block_time: time_receiver, }; let mock_chain_tip_sender = MockChainTipSender { best_tip_height: height_sender, + best_tip_hash: hash_sender, best_tip_block_time: time_sender, }; @@ -57,7 +66,14 @@ impl ChainTip for MockChainTip { } fn best_tip_hash(&self) -> Option { - unreachable!("Method not used in tests"); + *self.best_tip_hash.borrow() + } + + fn best_tip_height_and_hash(&self) -> Option<(block::Height, block::Hash)> { + let height = (*self.best_tip_height.borrow())?; + let hash = (*self.best_tip_hash.borrow())?; + + Some((height, hash)) } fn best_tip_block_time(&self) -> Option> { @@ -84,6 +100,13 @@ impl MockChainTipSender { .expect("attempt to send a best tip height to a dropped `MockChainTip`"); } + /// Send a new best tip hash to the [`MockChainTip`]. + pub fn send_best_tip_hash(&self, hash: impl Into>) { + self.best_tip_hash + .send(hash.into()) + .expect("attempt to send a best tip hash to a dropped `MockChainTip`"); + } + /// Send a new best tip block time to the [`MockChainTip`]. pub fn send_best_tip_block_time(&self, block_time: impl Into>>) { self.best_tip_block_time diff --git a/zebra-chain/src/parameters/network.rs b/zebra-chain/src/parameters/network.rs index 4d24ddf0..1cf127e1 100644 --- a/zebra-chain/src/parameters/network.rs +++ b/zebra-chain/src/parameters/network.rs @@ -101,6 +101,15 @@ impl Network { (canopy_activation + ZIP_212_GRACE_PERIOD_DURATION) .expect("ZIP-212 grace period ends at a valid block height") } + + /// Return the network name as defined in + /// [BIP70](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki#paymentdetailspaymentrequest) + pub fn bip70_network_name(&self) -> String { + match self { + Network::Mainnet => "main".to_string(), + Network::Testnet => "test".to_string(), + } + } } impl Default for Network { diff --git a/zebra-chain/src/parameters/network_upgrade.rs b/zebra-chain/src/parameters/network_upgrade.rs index bf3452f0..ac4be740 100644 --- a/zebra-chain/src/parameters/network_upgrade.rs +++ b/zebra-chain/src/parameters/network_upgrade.rs @@ -6,9 +6,11 @@ use crate::block; use crate::parameters::{Network, Network::*}; use std::collections::{BTreeMap, HashMap}; +use std::fmt; use std::ops::Bound::*; use chrono::{DateTime, Duration, Utc}; +use hex::{FromHex, ToHex}; #[cfg(any(test, feature = "proptest-impl"))] use proptest_derive::Arbitrary; @@ -118,15 +120,60 @@ const FAKE_TESTNET_ACTIVATION_HEIGHTS: &[(block::Height, NetworkUpgrade)] = &[ /// The Consensus Branch Id, used to bind transactions and blocks to a /// particular network upgrade. -#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] pub struct ConsensusBranchId(u32); +impl ConsensusBranchId { + /// Return the hash bytes in big-endian byte-order suitable for printing out byte by byte. + /// + /// Zebra displays consensus branch IDs in big-endian byte-order, + /// following the convention set by zcashd. + fn bytes_in_display_order(&self) -> [u8; 4] { + self.0.to_be_bytes() + } +} + impl From for u32 { fn from(branch: ConsensusBranchId) -> u32 { branch.0 } } +impl ToHex for &ConsensusBranchId { + fn encode_hex>(&self) -> T { + self.bytes_in_display_order().encode_hex() + } + + fn encode_hex_upper>(&self) -> T { + self.bytes_in_display_order().encode_hex_upper() + } +} + +impl ToHex for ConsensusBranchId { + fn encode_hex>(&self) -> T { + self.bytes_in_display_order().encode_hex() + } + + fn encode_hex_upper>(&self) -> T { + self.bytes_in_display_order().encode_hex_upper() + } +} + +impl FromHex for ConsensusBranchId { + type Error = <[u8; 4] as FromHex>::Error; + + fn from_hex>(hex: T) -> Result { + let branch = <[u8; 4]>::from_hex(hex)?; + Ok(ConsensusBranchId(u32::from_be_bytes(branch))) + } +} + +impl fmt::Display for ConsensusBranchId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(&self.encode_hex::()) + } +} + /// Network Upgrade Consensus Branch Ids. /// /// Branch ids are the same for mainnet and testnet. If there is a testnet @@ -175,8 +222,8 @@ const TESTNET_MINIMUM_DIFFICULTY_START_HEIGHT: block::Height = block::Height(299 pub const TESTNET_MAX_TIME_START_HEIGHT: block::Height = block::Height(653_606); impl NetworkUpgrade { - /// Returns a BTreeMap of activation heights and network upgrades for - /// `network`. + /// Returns a map between activation heights and network upgrades for `network`, + /// in ascending height order. /// /// If the activation height of a future upgrade is not known, that /// network upgrade does not appear in the list. @@ -186,7 +233,7 @@ impl NetworkUpgrade { /// When the environment variable TEST_FAKE_ACTIVATION_HEIGHTS is set /// and it's a test build, this returns a list of fake activation heights /// used by some tests. - pub(crate) fn activation_list(network: Network) -> BTreeMap { + pub fn activation_list(network: Network) -> BTreeMap { let (mainnet_heights, testnet_heights) = { #[cfg(not(feature = "zebra-test"))] { @@ -263,7 +310,7 @@ impl NetworkUpgrade { NetworkUpgrade::activation_list(network).contains_key(&height) } - /// Returns a BTreeMap of NetworkUpgrades and their ConsensusBranchIds. + /// Returns an unordered mapping between NetworkUpgrades and their ConsensusBranchIds. /// /// Branch ids are the same for mainnet and testnet. /// @@ -410,6 +457,16 @@ impl NetworkUpgrade { } impl ConsensusBranchId { + /// The value used by `zcashd` RPCs for missing consensus branch IDs. + /// + /// # Consensus + /// + /// This value must only be used in RPCs. + /// + /// The consensus rules handle missing branch IDs by rejecting blocks and transactions, + /// so this substitute value must not be used in consensus-critical code. + pub const RPC_MISSING_ID: ConsensusBranchId = ConsensusBranchId(0); + /// Returns the current consensus branch id for `network` and `height`. /// /// Returns None if the network has no branch id at this height. diff --git a/zebra-chain/src/parameters/tests.rs b/zebra-chain/src/parameters/tests.rs index 166ab31a..a5a62d5b 100644 --- a/zebra-chain/src/parameters/tests.rs +++ b/zebra-chain/src/parameters/tests.rs @@ -233,3 +233,21 @@ fn branch_id_consistent(network: Network) { } } } + +// TODO: split this file in unit.rs and prop.rs +use hex::{FromHex, ToHex}; +use proptest::prelude::*; + +proptest! { + #[test] + fn branch_id_hex_roundtrip(nu in any::()) { + zebra_test::init(); + + if let Some(branch) = nu.branch_id() { + let hex_branch: String = branch.encode_hex(); + let new_branch = ConsensusBranchId::from_hex(hex_branch.clone()).expect("hex branch_id should parse"); + prop_assert_eq!(branch, new_branch); + prop_assert_eq!(hex_branch, new_branch.to_string()); + } + } +} diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index f68d3627..3620e453 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -13,6 +13,7 @@ zebra-network = { path = "../zebra-network" } zebra-node-services = { path = "../zebra-node-services" } zebra-state = { path = "../zebra-state" } +chrono = "0.4.19" futures = "0.3.21" # lightwalletd sends JSON-RPC requests over HTTP 1.1 @@ -21,6 +22,9 @@ hyper = { version = "0.14.17", features = ["http1", "server"] } jsonrpc-core = "18.0.0" jsonrpc-derive = "18.0.0" jsonrpc-http-server = "18.0.0" +# zebra-rpc needs the preserve_order feature in serde_json, which is a dependency of jsonrpc-core +serde_json = { version = "1.0.79", features = ["preserve_order"] } +indexmap = { version = "1.8.0", features = ["serde"] } tokio = { version = "1.17.0", features = ["time", "rt-multi-thread", "macros", "tracing"] } tower = "0.4.12" diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index abd84efb..6e119afa 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -4,20 +4,22 @@ //! as used by `lightwalletd.` //! //! Some parts of the `zcashd` RPC documentation are outdated. -//! So this implementation follows the `lightwalletd` client implementation. +//! So this implementation follows the `zcashd` server and `lightwalletd` client implementations. use std::{collections::HashSet, io, sync::Arc}; +use chrono::Utc; use futures::{FutureExt, TryFutureExt}; use hex::{FromHex, ToHex}; +use indexmap::IndexMap; use jsonrpc_core::{self, BoxFuture, Error, ErrorCode, Result}; use jsonrpc_derive::rpc; use tower::{buffer::Buffer, Service, ServiceExt}; use zebra_chain::{ - block::{self, SerializedBlock}, + block::{self, Height, SerializedBlock}, chain_tip::ChainTip, - parameters::Network, + parameters::{ConsensusBranchId, Network, NetworkUpgrade}, serialization::{SerializationError, ZcashDeserialize}, transaction::{self, SerializedTransaction, Transaction}, }; @@ -49,9 +51,10 @@ pub trait Rpc { /// /// zcashd reference: [`getblockchaininfo`](https://zcash.github.io/rpc/getblockchaininfo.html) /// - /// TODO in the context of https://github.com/ZcashFoundation/zebra/issues/3143: - /// - list the arguments and fields that lightwalletd uses - /// - note any other lightwalletd changes + /// # Notes + /// + /// Some fields from the zcashd reference are missing from Zebra's [`GetBlockChainInfo`]. It only contains the fields + /// [required for lightwalletd support.](https://github.com/zcash/lightwalletd/blob/v0.4.9/common/common.go#L72-L89) #[rpc(name = "getblockchaininfo")] fn get_blockchain_info(&self) -> Result; @@ -216,11 +219,96 @@ where } fn get_blockchain_info(&self) -> Result { - // TODO: dummy output data, fix in the context of #3143 - // use self.latest_chain_tip.estimate_network_chain_tip_height() - // to estimate the current block height on the network + let network = self.network; + + // `chain` field + let chain = self.network.bip70_network_name(); + + // `blocks` and `best_block_hash` fields + let (tip_height, tip_hash) = self + .latest_chain_tip + .best_tip_height_and_hash() + .ok_or_else(|| Error { + code: ErrorCode::ServerError(0), + message: "No Chain tip available yet".to_string(), + data: None, + })?; + + // `estimated_height` field + let current_block_time = + self.latest_chain_tip + .best_tip_block_time() + .ok_or_else(|| Error { + code: ErrorCode::ServerError(0), + message: "No Chain tip available yet".to_string(), + data: None, + })?; + + let zebra_estimated_height = self + .latest_chain_tip + .estimate_network_chain_tip_height(network, Utc::now()) + .ok_or_else(|| Error { + code: ErrorCode::ServerError(0), + message: "No Chain tip available yet".to_string(), + data: None, + })?; + + let estimated_height = + if current_block_time > Utc::now() || zebra_estimated_height < tip_height { + tip_height + } else { + zebra_estimated_height + }; + + // `upgrades` object + // + // Get the network upgrades in height order, like `zcashd`. + let mut upgrades = IndexMap::new(); + for (activation_height, network_upgrade) in NetworkUpgrade::activation_list(network) { + // Zebra defines network upgrades based on incompatible consensus rule changes, + // but zcashd defines them based on ZIPs. + // + // All the network upgrades with a consensus branch ID are the same in Zebra and zcashd. + if let Some(branch_id) = network_upgrade.branch_id() { + // zcashd's RPC seems to ignore Disabled network upgrades, so Zebra does too. + let status = if tip_height >= activation_height { + NetworkUpgradeStatus::Active + } else { + NetworkUpgradeStatus::Pending + }; + + let upgrade = NetworkUpgradeInfo { + name: network_upgrade, + activation_height, + status, + }; + upgrades.insert(ConsensusBranchIdHex(branch_id), upgrade); + } + } + + // `consensus` object + let next_block_height = + (tip_height + 1).expect("valid chain tips are a lot less than Height::MAX"); + let consensus = TipConsensusBranch { + chain_tip: ConsensusBranchIdHex( + NetworkUpgrade::current(network, tip_height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID), + ), + next_block: ConsensusBranchIdHex( + NetworkUpgrade::current(network, next_block_height) + .branch_id() + .unwrap_or(ConsensusBranchId::RPC_MISSING_ID), + ), + }; + let response = GetBlockChainInfo { - chain: "TODO: main".to_string(), + chain, + blocks: tip_height.0, + best_block_hash: GetBestBlockHash(tip_hash), + estimated_height: estimated_height.0, + upgrades, + consensus, }; Ok(response) @@ -432,44 +520,85 @@ where } } -#[derive(serde::Serialize, serde::Deserialize)] /// Response to a `getinfo` RPC request. /// /// See the notes for the [`Rpc::get_info` method]. +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub struct GetInfo { build: String, subversion: String, } -#[derive(serde::Serialize, serde::Deserialize)] /// Response to a `getblockchaininfo` RPC request. /// /// See the notes for the [`Rpc::get_blockchain_info` method]. +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub struct GetBlockChainInfo { chain: String, - // TODO: add other fields used by lightwalletd (#3143) + blocks: u32, + #[serde(rename = "bestblockhash")] + best_block_hash: GetBestBlockHash, + #[serde(rename = "estimatedheight")] + estimated_height: u32, + upgrades: IndexMap, + consensus: TipConsensusBranch, +} + +/// A hex-encoded [`ConsensusBranchId`] string. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] +struct ConsensusBranchIdHex(#[serde(with = "hex")] ConsensusBranchId); + +/// Information about [`NetworkUpgrade`] activation. +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +struct NetworkUpgradeInfo { + name: NetworkUpgrade, + #[serde(rename = "activationheight")] + activation_height: Height, + status: NetworkUpgradeStatus, +} + +/// The activation status of a [`NetworkUpgrade`]. +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +enum NetworkUpgradeStatus { + #[serde(rename = "active")] + Active, + #[serde(rename = "disabled")] + Disabled, + #[serde(rename = "pending")] + Pending, +} + +/// The [`ConsensusBranchId`]s for the tip and the next block. +/// +/// These branch IDs are different when the next block is a network upgrade activation block. +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +struct TipConsensusBranch { + #[serde(rename = "chaintip")] + chain_tip: ConsensusBranchIdHex, + #[serde(rename = "nextblock")] + next_block: ConsensusBranchIdHex, } -#[derive(Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] /// Response to a `sendrawtransaction` RPC request. /// /// Contains the hex-encoded hash of the sent transaction. /// /// See the notes for the [`Rpc::send_raw_transaction` method]. +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub struct SentTransactionHash(#[serde(with = "hex")] transaction::Hash); -#[derive(serde::Serialize)] /// Response to a `getblock` RPC request. /// /// See the notes for the [`Rpc::get_block` method]. +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)] pub struct GetBlock(#[serde(with = "hex")] SerializedBlock); -#[derive(Debug, PartialEq, serde::Serialize)] /// Response to a `getbestblockhash` RPC request. /// /// Contains the hex-encoded hash of the tip block. /// /// Also see the notes for the [`Rpc::get_best_block_hash` method]. +#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] pub struct GetBestBlockHash(#[serde(with = "hex")] block::Hash); /// Response to a `getrawtransaction` RPC request. diff --git a/zebra-rpc/src/methods/tests/prop.rs b/zebra-rpc/src/methods/tests/prop.rs index 26fb1c07..aac81f0c 100644 --- a/zebra-rpc/src/methods/tests/prop.rs +++ b/zebra-rpc/src/methods/tests/prop.rs @@ -9,8 +9,12 @@ use thiserror::Error; use tower::buffer::Buffer; use zebra_chain::{ - chain_tip::NoChainTip, - parameters::Network::*, + block::{Block, Height}, + chain_tip::{mock::MockChainTip, NoChainTip}, + parameters::{ + Network::{self, *}, + NetworkUpgrade, + }, serialization::{ZcashDeserialize, ZcashSerialize}, transaction::{self, Transaction, UnminedTx, UnminedTxId}, }; @@ -19,7 +23,7 @@ use zebra_state::BoxError; use zebra_test::mock_service::MockService; -use super::super::{Rpc, RpcImpl, SentTransactionHash}; +use super::super::{NetworkUpgradeStatus, Rpc, RpcImpl, SentTransactionHash}; proptest! { /// Test that when sending a raw transaction, it is received by the mempool service. @@ -416,6 +420,94 @@ proptest! { ), "Result is not an invalid parameters error: {result:?}" ); + Ok::<_, TestCaseError>(()) + })?; + } + + /// Test the `get_blockchain_info` response when Zebra's state is empty. + #[test] + fn get_blockchain_info_response_without_a_chain_tip(network in any::()) { + let runtime = zebra_test::init_async(); + let _guard = runtime.enter(); + let mut mempool = MockService::build().for_prop_tests(); + let mut state: MockService<_, _, _, BoxError> = MockService::build().for_prop_tests(); + // look for an error with a `NoChainTip` + let rpc = RpcImpl::new( + "RPC test", + Buffer::new(mempool.clone(), 1), + Buffer::new(state.clone(), 1), + NoChainTip, + network, + ); + let response = rpc.get_blockchain_info(); + prop_assert_eq!(&response.err().unwrap().message, "No Chain tip available yet"); + runtime.block_on(async move { + mempool.expect_no_requests().await?; + state.expect_no_requests().await?; + Ok::<_, TestCaseError>(()) + })?; + } + + /// Test the `get_blockchain_info` response using an arbitrary block as the `ChainTip`. + #[test] + fn get_blockchain_info_response_with_an_arbitrary_chain_tip( + network in any::(), + block in any::(), + ) { + let runtime = zebra_test::init_async(); + let _guard = runtime.enter(); + let mut mempool = MockService::build().for_prop_tests(); + let mut state: MockService<_, _, _, BoxError> = MockService::build().for_prop_tests(); + + // get block data + let block_height = block.coinbase_height().unwrap(); + let block_hash = block.hash(); + let block_time = block.header.time; + + // create a mocked `ChainTip` + let (chain_tip, mock_chain_tip_sender) = MockChainTip::new(); + mock_chain_tip_sender.send_best_tip_height(block_height); + mock_chain_tip_sender.send_best_tip_hash(block_hash); + mock_chain_tip_sender.send_best_tip_block_time(block_time); + + // Start RPC with the mocked `ChainTip` + let rpc = RpcImpl::new( + "RPC test", + Buffer::new(mempool.clone(), 1), + Buffer::new(state.clone(), 1), + chain_tip, + network, + ); + let response = rpc.get_blockchain_info(); + + // Check response + match response { + Ok(info) => { + prop_assert_eq!(info.chain, network.bip70_network_name()); + prop_assert_eq!(info.blocks, block_height.0); + prop_assert_eq!(info.best_block_hash.0, block_hash); + prop_assert!(info.estimated_height < Height::MAX.0); + + prop_assert_eq!(info.consensus.chain_tip.0, NetworkUpgrade::current(network, block_height).branch_id().unwrap()); + prop_assert_eq!(info.consensus.next_block.0, NetworkUpgrade::current(network, (block_height + 1).unwrap()).branch_id().unwrap()); + + for u in info.upgrades { + let mut status = NetworkUpgradeStatus::Active; + if block_height < u.1.activation_height { + status = NetworkUpgradeStatus::Pending; + } + prop_assert_eq!(u.1.status, status); + } + }, + Err(_) => { + unreachable!("Test should never error with the data we are feeding it") + }, + }; + + // check no requests were made during this test + runtime.block_on(async move { + mempool.expect_no_requests().await?; + state.expect_no_requests().await?; Ok::<_, TestCaseError>(()) })?; diff --git a/zebra-state/src/service/chain_tip.rs b/zebra-state/src/service/chain_tip.rs index 7ed7d32a..9e07b396 100644 --- a/zebra-state/src/service/chain_tip.rs +++ b/zebra-state/src/service/chain_tip.rs @@ -333,6 +333,11 @@ impl ChainTip for LatestChainTip { self.with_chain_tip_block(|block| block.hash) } + #[instrument(skip(self))] + fn best_tip_height_and_hash(&self) -> Option<(block::Height, block::Hash)> { + self.with_chain_tip_block(|block| (block.height, block.hash)) + } + #[instrument(skip(self))] fn best_tip_block_time(&self) -> Option> { self.with_chain_tip_block(|block| block.time) diff --git a/zebra-test/src/command.rs b/zebra-test/src/command.rs index 6d0561c0..77237cca 100644 --- a/zebra-test/src/command.rs +++ b/zebra-test/src/command.rs @@ -104,6 +104,7 @@ impl CommandExt for Command { stdout: None, stderr: None, failure_regexes: RegexSet::empty(), + ignore_regexes: RegexSet::empty(), deadline: None, bypass_test_capture: false, }) @@ -204,6 +205,15 @@ pub struct TestChild { /// If any line matches any failure regex, the test fails. failure_regexes: RegexSet, + /// Command outputs which are ignored when checking for test failure. + /// These regexes override `failure_regexes`. + /// + /// This list of regexes is matches against `stdout` or `stderr`, + /// in every method that reads command output. + /// + /// If a line matches any ignore regex, the failure regex check is skipped for that line. + ignore_regexes: RegexSet, + /// The deadline for this command to finish. /// /// Only checked when the command outputs each new line (#1140). @@ -215,23 +225,51 @@ pub struct TestChild { } /// Checks command output log `line` from `cmd` against a `failure_regexes` regex set, -/// and panics if any regex matches the log line. +/// and panics if any regex matches. The line is skipped if it matches `ignore_regexes`. /// /// # Panics /// -/// - if any stdout or stderr lines match any failure regex +/// - if any stdout or stderr lines match any failure regex, but do not match any ignore regex pub fn check_failure_regexes( line: &std::io::Result, failure_regexes: &RegexSet, + ignore_regexes: &RegexSet, cmd: &str, + bypass_test_capture: bool, ) { if let Ok(line) = line { + let ignore_matches = ignore_regexes.matches(line); + let ignore_matches: Vec<&str> = ignore_matches + .iter() + .map(|index| ignore_regexes.patterns()[index].as_str()) + .collect(); + let failure_matches = failure_regexes.matches(line); let failure_matches: Vec<&str> = failure_matches .iter() .map(|index| failure_regexes.patterns()[index].as_str()) .collect(); + if !ignore_matches.is_empty() { + let ignore_matches = ignore_matches.join(","); + + let ignore_msg = if failure_matches.is_empty() { + format!( + "Log matched ignore regexes: {:?}, but no failure regexes", + ignore_matches, + ) + } else { + let failure_matches = failure_matches.join(","); + format!( + "Ignoring failure regexes: {:?}, because log matched ignore regexes: {:?}", + failure_matches, ignore_matches, + ) + }; + + write_to_test_logs(ignore_msg, bypass_test_capture); + return; + } + assert!( failure_matches.is_empty(), "test command:\n\ @@ -247,64 +285,124 @@ pub fn check_failure_regexes( } } +/// Write `line` to stdout, so it can be seen in the test logs. +/// +/// Set `bypass_test_capture` to `true` or +/// use `cargo test -- --nocapture` to see this output. +/// +/// May cause weird reordering for stdout / stderr. +/// Uses stdout even if the original lines were from stderr. +#[allow(clippy::print_stdout)] +fn write_to_test_logs(line: S, bypass_test_capture: bool) +where + S: AsRef, +{ + let line = line.as_ref(); + + if bypass_test_capture { + // Send lines directly to the terminal (or process stdout file redirect). + #[allow(clippy::explicit_write)] + writeln!(std::io::stdout(), "{}", line).unwrap(); + } else { + // If the test fails, the test runner captures and displays this output. + // To show this output unconditionally, use `cargo test -- --nocapture`. + println!("{}", line); + } + + // Some OSes require a flush to send all output to the terminal. + let _ = std::io::stdout().lock().flush(); +} + +/// A [`CollectRegexSet`] iterator that never matches anything. +/// +/// Used to work around type inference issues in [`TestChild::with_failure_regex_iter`]. +pub const NO_MATCHES_REGEX_ITER: &[&str] = &[]; + impl TestChild { - /// Sets up command output so it is checked against a failure regex set. + /// Sets up command output so each line is checked against a failure regex set, + /// unless it matches any of the ignore regexes. + /// /// The failure regexes are ignored by `wait_with_output`. /// - /// [`TestChild::with_failure_regexes`] wrapper for strings, [`Regex`]es, - /// and [`RegexSet`]s. + /// To never match any log lines, use `RegexSet::empty()`. + /// + /// This method is a [`TestChild::with_failure_regexes`] wrapper for + /// strings, [`Regex`]es, and [`RegexSet`]s. /// /// # Panics /// /// - adds a panic to any method that reads output, /// if any stdout or stderr lines match any failure regex - pub fn with_failure_regex_set(self, failure_regexes: R) -> Self + pub fn with_failure_regex_set(self, failure_regexes: F, ignore_regexes: X) -> Self where - R: ToRegexSet, + F: ToRegexSet, + X: ToRegexSet, { let failure_regexes = failure_regexes .to_regex_set() - .expect("regexes must be valid"); + .expect("failure regexes must be valid"); - self.with_failure_regexes(failure_regexes) + let ignore_regexes = ignore_regexes + .to_regex_set() + .expect("ignore regexes must be valid"); + + self.with_failure_regexes(failure_regexes, ignore_regexes) } - /// Sets up command output so it is checked against a failure regex set. + /// Sets up command output so each line is checked against a failure regex set, + /// unless it matches any of the ignore regexes. + /// /// The failure regexes are ignored by `wait_with_output`. /// - /// [`TestChild::with_failure_regexes`] wrapper for regular expression iterators. + /// To never match any log lines, use [`NO_MATCHES_REGEX_ITER`]. + /// + /// This method is a [`TestChild::with_failure_regexes`] wrapper for + /// regular expression iterators. /// /// # Panics /// /// - adds a panic to any method that reads output, /// if any stdout or stderr lines match any failure regex - pub fn with_failure_regex_iter(self, failure_regexes: I) -> Self + pub fn with_failure_regex_iter(self, failure_regexes: F, ignore_regexes: X) -> Self where - I: CollectRegexSet, + F: CollectRegexSet, + X: CollectRegexSet, { let failure_regexes = failure_regexes .collect_regex_set() - .expect("regexes must be valid"); + .expect("failure regexes must be valid"); - self.with_failure_regexes(failure_regexes) + let ignore_regexes = ignore_regexes + .collect_regex_set() + .expect("ignore regexes must be valid"); + + self.with_failure_regexes(failure_regexes, ignore_regexes) } - /// Sets up command output so it is checked against a failure regex set. + /// Sets up command output so each line is checked against a failure regex set, + /// unless it matches any of the ignore regexes. + /// /// The failure regexes are ignored by `wait_with_output`. /// /// # Panics /// /// - adds a panic to any method that reads output, /// if any stdout or stderr lines match any failure regex - pub fn with_failure_regexes(mut self, failure_regexes: RegexSet) -> Self { + pub fn with_failure_regexes( + mut self, + failure_regexes: RegexSet, + ignore_regexes: impl Into>, + ) -> Self { self.failure_regexes = failure_regexes; + self.ignore_regexes = ignore_regexes.into().unwrap_or_else(RegexSet::empty); self.apply_failure_regexes_to_outputs(); self } - /// Applies the failure regex set to command output. + /// Applies the failure and ignore regex sets to command output. + /// /// The failure regexes are ignored by `wait_with_output`. /// /// # Panics @@ -329,7 +427,8 @@ impl TestChild { } } - /// Maps a reader into a string line iterator. + /// Maps a reader into a string line iterator, + /// and applies the failure and ignore regex sets to it. fn map_into_string_lines( &self, reader: R, @@ -338,11 +437,20 @@ impl TestChild { R: Read + Debug + 'static, { let failure_regexes = self.failure_regexes.clone(); + let ignore_regexes = self.ignore_regexes.clone(); let cmd = self.cmd.clone(); + let bypass_test_capture = self.bypass_test_capture; let reader = BufReader::new(reader); - let lines = BufRead::lines(reader) - .inspect(move |line| check_failure_regexes(line, &failure_regexes, &cmd)); + let lines = BufRead::lines(reader).inspect(move |line| { + check_failure_regexes( + line, + &failure_regexes, + &ignore_regexes, + &cmd, + bypass_test_capture, + ) + }); Box::new(lines) as _ } @@ -385,6 +493,7 @@ impl TestChild { while self.wait_for_stdout_line(None) {} if wrote_lines { + // Write an empty line, to make output more readable self.write_to_test_logs(""); } } @@ -683,20 +792,7 @@ impl TestChild { where S: AsRef, { - let line = line.as_ref(); - - if self.bypass_test_capture { - // Send lines directly to the terminal (or process stdout file redirect). - #[allow(clippy::explicit_write)] - writeln!(std::io::stdout(), "{}", line).unwrap(); - } else { - // If the test fails, the test runner captures and displays this output. - // To show this output unconditionally, use `cargo test -- --nocapture`. - println!("{}", line); - } - - // Some OSes require a flush to send all output to the terminal. - let _ = std::io::stdout().lock().flush(); + write_to_test_logs(line, self.bypass_test_capture); } /// Kill `child`, wait for its output, and use that output as the context for diff --git a/zebra-test/tests/command.rs b/zebra-test/tests/command.rs index 0025750c..9f3d4eee 100644 --- a/zebra-test/tests/command.rs +++ b/zebra-test/tests/command.rs @@ -1,9 +1,13 @@ use std::{process::Command, time::Duration}; use color_eyre::eyre::{eyre, Result}; +use regex::RegexSet; use tempfile::tempdir; -use zebra_test::{command::TestDirExt, prelude::Stdio}; +use zebra_test::{ + command::{TestDirExt, NO_MATCHES_REGEX_ITER}, + prelude::Stdio, +}; /// Returns true if `cmd` with `args` runs successfully. /// @@ -200,7 +204,7 @@ fn failure_regex_matches_stdout_failure_message() { .spawn_child_with_command(TEST_CMD, &["failure_message"]) .unwrap() .with_timeout(Duration::from_secs(2)) - .with_failure_regex_set("fail"); + .with_failure_regex_set("fail", RegexSet::empty()); // Any method that reads output should work here. // We use a non-matching regex, to trigger the failure panic. @@ -236,7 +240,7 @@ fn failure_regex_matches_stderr_failure_message() { .spawn_child_with_command(TEST_CMD, &["-c", "read -t 1 -p failure_message"]) .unwrap() .with_timeout(Duration::from_secs(5)) - .with_failure_regex_set("fail"); + .with_failure_regex_set("fail", RegexSet::empty()); // Any method that reads output should work here. // We use a non-matching regex, to trigger the failure panic. @@ -266,7 +270,7 @@ fn failure_regex_matches_stdout_failure_message_drop() { .spawn_child_with_command(TEST_CMD, &["failure_message"]) .unwrap() .with_timeout(Duration::from_secs(5)) - .with_failure_regex_set("fail"); + .with_failure_regex_set("fail", RegexSet::empty()); // Give the child process enough time to print its output. std::thread::sleep(Duration::from_secs(1)); @@ -295,7 +299,7 @@ fn failure_regex_matches_stdout_failure_message_kill() { .spawn_child_with_command(TEST_CMD, &["failure_message"]) .unwrap() .with_timeout(Duration::from_secs(5)) - .with_failure_regex_set("fail"); + .with_failure_regex_set("fail", RegexSet::empty()); // Give the child process enough time to print its output. std::thread::sleep(Duration::from_secs(1)); @@ -326,7 +330,7 @@ fn failure_regex_matches_stdout_failure_message_kill_on_error() { .spawn_child_with_command(TEST_CMD, &["failure_message"]) .unwrap() .with_timeout(Duration::from_secs(5)) - .with_failure_regex_set("fail"); + .with_failure_regex_set("fail", RegexSet::empty()); // Give the child process enough time to print its output. std::thread::sleep(Duration::from_secs(1)); @@ -358,7 +362,7 @@ fn failure_regex_matches_stdout_failure_message_no_kill_on_error() { .spawn_child_with_command(TEST_CMD, &["failure_message"]) .unwrap() .with_timeout(Duration::from_secs(5)) - .with_failure_regex_set("fail"); + .with_failure_regex_set("fail", RegexSet::empty()); // Give the child process enough time to print its output. std::thread::sleep(Duration::from_secs(1)); @@ -397,7 +401,7 @@ fn failure_regex_timeout_continuous_output() { .spawn_child_with_command(TEST_CMD, &["-v", "/dev/zero"]) .unwrap() .with_timeout(Duration::from_secs(2)) - .with_failure_regex_set("0"); + .with_failure_regex_set("0", RegexSet::empty()); // We need to use expect_stdout_line_matches, because wait_with_output ignores timeouts. // We use a non-matching regex, to trigger the timeout and the failure panic. @@ -429,7 +433,7 @@ fn failure_regex_matches_stdout_failure_message_wait_for_output() { .spawn_child_with_command(TEST_CMD, &["failure_message"]) .unwrap() .with_timeout(Duration::from_secs(5)) - .with_failure_regex_set("fail"); + .with_failure_regex_set("fail", RegexSet::empty()); // Give the child process enough time to print its output. std::thread::sleep(Duration::from_secs(1)); @@ -438,3 +442,80 @@ fn failure_regex_matches_stdout_failure_message_wait_for_output() { // or the output should be read on drop. child.wait_with_output().unwrap_err(); } + +/// Make sure failure regex iters detect when a child process prints a failure message to stdout, +/// and panic with a test failure message. +#[test] +#[should_panic(expected = "Logged a failure message")] +fn failure_regex_iter_matches_stdout_failure_message() { + zebra_test::init(); + + const TEST_CMD: &str = "echo"; + // Skip the test if the test system does not have the command + if !is_command_available(TEST_CMD, &[]) { + panic!( + "skipping test: command not available\n\ + fake panic message: Logged a failure message" + ); + } + + let mut child = tempdir() + .unwrap() + .spawn_child_with_command(TEST_CMD, &["failure_message"]) + .unwrap() + .with_timeout(Duration::from_secs(2)) + .with_failure_regex_iter( + ["fail"].iter().cloned(), + NO_MATCHES_REGEX_ITER.iter().cloned(), + ); + + // Any method that reads output should work here. + // We use a non-matching regex, to trigger the failure panic. + child + .expect_stdout_line_matches("this regex should not match") + .unwrap_err(); +} + +/// Make sure ignore regexes override failure regexes. +#[test] +fn ignore_regex_ignores_stdout_failure_message() { + zebra_test::init(); + + const TEST_CMD: &str = "echo"; + // Skip the test if the test system does not have the command + if !is_command_available(TEST_CMD, &[]) { + return; + } + + let mut child = tempdir() + .unwrap() + .spawn_child_with_command(TEST_CMD, &["failure_message ignore_message"]) + .unwrap() + .with_timeout(Duration::from_secs(2)) + .with_failure_regex_set("fail", "ignore"); + + // Any method that reads output should work here. + child.expect_stdout_line_matches("ignore_message").unwrap(); +} + +/// Make sure ignore regex iters override failure regex iters. +#[test] +fn ignore_regex_iter_ignores_stdout_failure_message() { + zebra_test::init(); + + const TEST_CMD: &str = "echo"; + // Skip the test if the test system does not have the command + if !is_command_available(TEST_CMD, &[]) { + return; + } + + let mut child = tempdir() + .unwrap() + .spawn_child_with_command(TEST_CMD, &["failure_message ignore_message"]) + .unwrap() + .with_timeout(Duration::from_secs(2)) + .with_failure_regex_iter(["fail"].iter().cloned(), ["ignore"].iter().cloned()); + + // Any method that reads output should work here. + child.expect_stdout_line_matches("ignore_message").unwrap(); +} diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index ba6320da..4ad4e5df 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -61,7 +61,8 @@ abscissa_core = { version = "0.5", features = ["testing"] } once_cell = "1.10.0" regex = "1.5.5" semver = "1.0.6" -serde_json = "1.0" +# zebra-rpc needs the preserve_order feature, it also makes test results more stable +serde_json = { version = "1.0.79", features = ["preserve_order"] } tempfile = "3.3.0" tokio = { version = "1.17.0", features = ["full", "test-util"] } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index cf5e9ca9..00bbee3a 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -35,7 +35,11 @@ use zebra_chain::{ use zebra_network::constants::PORT_IN_USE_ERROR; use zebra_state::constants::LOCK_FILE_ERROR; -use zebra_test::{command::ContextFrom, net::random_known_port, prelude::*}; +use zebra_test::{ + command::{ContextFrom, NO_MATCHES_REGEX_ITER}, + net::random_known_port, + prelude::*, +}; mod common; @@ -989,6 +993,9 @@ async fn rpc_endpoint() -> Result<()> { } /// Failure log messages for any process, from the OS or shell. +/// +/// These messages show that the child process has failed. +/// So when we see them in the logs, we make the test fail. const PROCESS_FAILURE_MESSAGES: &[&str] = &[ // Linux "Aborted", @@ -998,6 +1005,9 @@ const PROCESS_FAILURE_MESSAGES: &[&str] = &[ ]; /// Failure log messages from Zebra. +/// +/// These `zebrad` messages show that the `lightwalletd` integration test has failed. +/// So when we see them in the logs, we make the test fail. const ZEBRA_FAILURE_MESSAGES: &[&str] = &[ // Rust-specific panics "The application panicked", @@ -1020,6 +1030,9 @@ const ZEBRA_FAILURE_MESSAGES: &[&str] = &[ ]; /// Failure log messages from lightwalletd. +/// +/// These `lightwalletd` messages show that the `lightwalletd` integration test has failed. +/// So when we see them in the logs, we make the test fail. const LIGHTWALLETD_FAILURE_MESSAGES: &[&str] = &[ // Go-specific panics "panic:", @@ -1039,7 +1052,7 @@ const LIGHTWALLETD_FAILURE_MESSAGES: &[&str] = &[ // Go json package error messages: "json: cannot unmarshal", "into Go value of type", - // lightwalletd RPC error messages from: + // lightwalletd custom RPC error messages from: // https://github.com/adityapk00/lightwalletd/blob/master/common/common.go "block requested is newer than latest block", "Cache add failed", @@ -1077,6 +1090,21 @@ const LIGHTWALLETD_FAILURE_MESSAGES: &[&str] = &[ // get_address_utxos ]; +/// Ignored failure logs for lightwalletd. +/// These regexes override the [`LIGHTWALLETD_FAILURE_MESSAGES`]. +/// +/// These `lightwalletd` messages look like failure messages, but they are actually ok. +/// So when we see them in the logs, we make the test continue. +const LIGHTWALLETD_IGNORE_MESSAGES: &[&str] = &[ + // Exceptions to lightwalletd custom RPC error messages: + // + // This log matches the "error with" RPC error message, + // but we expect Zebra to start with an empty state. + // + // TODO: this exception should not be used for the cached state tests (#3511) + r#"No Chain tip available yet","level":"warning","msg":"error with getblockchaininfo rpc, retrying"#, +]; + /// Launch `zebrad` with an RPC port, and make sure `lightwalletd` works with Zebra. /// /// This test only runs when the `ZEBRA_TEST_LIGHTWALLETD` env var is set. @@ -1108,6 +1136,7 @@ fn lightwalletd_integration() -> Result<()> { .iter() .chain(PROCESS_FAILURE_MESSAGES) .cloned(), + NO_MATCHES_REGEX_ITER.iter().cloned(), ); // Wait until `zebrad` has opened the RPC endpoint @@ -1132,6 +1161,8 @@ fn lightwalletd_integration() -> Result<()> { .iter() .chain(PROCESS_FAILURE_MESSAGES) .cloned(), + // TODO: some exceptions do not apply to the cached state tests (#3511) + LIGHTWALLETD_IGNORE_MESSAGES.iter().cloned(), ); // Wait until `lightwalletd` has launched @@ -1142,28 +1173,42 @@ fn lightwalletd_integration() -> Result<()> { // getblockchaininfo // - // TODO: add correct sapling height, chain, branchID (PR #3891) + // TODO: update branchID when we're using cached state (#3511) // add "Waiting for zcashd height to reach Sapling activation height" - let result = lightwalletd.expect_stdout_line_matches("Got sapling height"); + let result = lightwalletd.expect_stdout_line_matches( + "Got sapling height 419200 block height [0-9]+ chain main branchID 00000000", + ); let (_, zebrad) = zebrad.kill_on_error(result)?; let result = lightwalletd.expect_stdout_line_matches("Found 0 blocks in cache"); let (_, zebrad) = zebrad.kill_on_error(result)?; - // getblock with block 1 in Zebra's state + // getblock with the first Sapling block in Zebra's state // // zcash/lightwalletd calls getbestblockhash here, but // adityapk00/lightwalletd calls getblock // - // Until block 1 has been downloaded, lightwalletd will log Zebra's RPC error: + // The log also depends on what is in Zebra's state: + // + // # Empty Zebra State + // + // lightwalletd tries to download the Sapling activation block, but it's not in the state. + // + // Until the Sapling activation block has been downloaded, lightwalletd will log Zebra's RPC error: // "error requesting block: 0: Block not found" - // But we can't check for that, because Zebra might download genesis before lightwalletd asks. // We also get a similar log when lightwalletd reaches the end of Zebra's cache. // - // After the first getblock call, lightwalletd will log: + // # Cached Zebra State + // + // After the first successful getblock call, lightwalletd will log: // "Block hash changed, clearing mempool clients" // But we can't check for that, because it can come before or after the Ingestor log. - let result = lightwalletd.expect_stdout_line_matches("Ingestor adding block to cache"); + // + // TODO: expect Ingestor log when we're using cached state (#3511) + // "Ingestor adding block to cache" + let result = lightwalletd.expect_stdout_line_matches( + r#"error requesting block: 0: Block not found","height":419200"#, + ); let (_, zebrad) = zebrad.kill_on_error(result)?; // (next RPC)