Add support for errors in zebra_test::Transcript (#678)

* Add support for errors in zebra_test::Transcript

* test transcript with an error checker

* switch to option instead of MockError

* update docs

* dont use verifier against ready_and

* cleanup exports and add docs

* handle todos

* fix doctest

* temp: use cleaner error handling example

* add ability to test only for presence of error

Cargo.lock

@@ -2747,6 +2747,7 @@ dependencies = [
  "futures",
  "hex",
  "lazy_static",
+ "thiserror",
  "tokio",
  "tower",
  "tracing",

zebra-state/tests/basic.rs

@@ -2,13 +2,12 @@ use color_eyre::eyre::Report;
 use once_cell::sync::Lazy;
 use std::sync::Arc;
 use tempdir::TempDir;
-
 use zebra_chain::{block::Block, serialization::ZcashDeserialize, Network, Network::*};
-use zebra_test::transcript::Transcript;
+use zebra_test::transcript::{TransError, Transcript};
 
 use zebra_state::*;
 
-static ADD_BLOCK_TRANSCRIPT: Lazy<Vec<(Request, Response)>> = Lazy::new(|| {
+static ADD_BLOCK_TRANSCRIPT: Lazy<Vec<(Request, Result<Response, TransError>)>> = Lazy::new(|| {
     let block: Arc<_> =
         Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_415000_BYTES[..])
             .unwrap()
@@ -19,13 +18,13 @@ static ADD_BLOCK_TRANSCRIPT: Lazy<Vec<(Request, Response)>> = Lazy::new(|| {
             Request::AddBlock {
                 block: block.clone(),
             },
-            Response::Added { hash },
+            Ok(Response::Added { hash }),
         ),
-        (Request::GetBlock { hash }, Response::Block { block }),
+        (Request::GetBlock { hash }, Ok(Response::Block { block })),
     ]
 });
 
-static GET_TIP_TRANSCRIPT: Lazy<Vec<(Request, Response)>> = Lazy::new(|| {
+static GET_TIP_TRANSCRIPT: Lazy<Vec<(Request, Result<Response, TransError>)>> = Lazy::new(|| {
     let block0: Arc<_> =
         Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..])
             .unwrap()
@@ -39,13 +38,13 @@ static GET_TIP_TRANSCRIPT: Lazy<Vec<(Request, Response)>> = Lazy::new(|| {
         // Insert higher block first, lower block second
         (
             Request::AddBlock { block: block1 },
-            Response::Added { hash: hash1 },
+            Ok(Response::Added { hash: hash1 }),
         ),
         (
             Request::AddBlock { block: block0 },
-            Response::Added { hash: hash0 },
+            Ok(Response::Added { hash: hash0 }),
         ),
-        (Request::GetTip, Response::Tip { hash: hash1 }),
+        (Request::GetTip, Ok(Response::Tip { hash: hash1 })),
     ]
 });
 

zebra-test/Cargo.toml

@@ -16,6 +16,7 @@ color-eyre = "0.5"
 tracing = "0.1.17"
 tracing-subscriber = "0.2.9"
 tracing-error = "0.1.2"
+thiserror = "1.0.20"
 
 [dev-dependencies]
 tokio = { version = "0.2", features = ["full"] }

zebra-test/src/transcript.rs

@@ -1,25 +1,66 @@
 //! A [`Service`](tower::Service) implementation based on a fixed transcript.
 
-use color_eyre::eyre::{ensure, eyre, Report};
+use color_eyre::{
+    eyre::{eyre, Report, WrapErr},
+    section::Section,
+    section::SectionExt,
+};
 use futures::future::{ready, Ready};
 use std::{
     fmt::Debug,
+    sync::Arc,
     task::{Context, Poll},
 };
 use tower::{Service, ServiceExt};
 
-type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
+type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
+
+pub type ErrorChecker = fn(Option<Error>) -> Result<(), Error>;
+
+#[derive(Debug, Clone)]
+pub enum TransError {
+    Any,
+    Exact(Arc<ErrorChecker>),
+}
+
+impl TransError {
+    pub fn exact(verifier: ErrorChecker) -> Self {
+        TransError::Exact(verifier.into())
+    }
+
+    fn check(&self, e: Error) -> Result<(), Report> {
+        match self {
+            TransError::Any => Ok(()),
+            TransError::Exact(checker) => checker(Some(e)),
+        }
+        .map_err(ErrorCheckerError)
+        .wrap_err("service returned an error but it didn't match the expected error")
+    }
+
+    fn mock(&self) -> Report {
+        match self {
+            TransError::Any => eyre!("mock error"),
+            TransError::Exact(checker) => checker(None).map_err(|e| eyre!(e)).expect_err(
+                "transcript should correctly produce the expected mock error when passed None",
+            ),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("ErrorChecker Error: {0}")]
+struct ErrorCheckerError(Error);
 
 pub struct Transcript<R, S, I>
 where
-    I: Iterator<Item = (R, S)>,
+    I: Iterator<Item = (R, Result<S, TransError>)>,
 {
     messages: I,
 }
 
 impl<R, S, I> From<I> for Transcript<R, S, I>
 where
-    I: Iterator<Item = (R, S)>,
+    I: Iterator<Item = (R, Result<S, TransError>)>,
 {
     fn from(messages: I) -> Self {
         Self { messages }
@@ -28,33 +69,72 @@ where
 
 impl<R, S, I> Transcript<R, S, I>
 where
-    I: Iterator<Item = (R, S)>,
+    I: Iterator<Item = (R, Result<S, TransError>)>,
     R: Debug,
     S: Debug + Eq,
 {
     pub async fn check<C>(mut self, mut to_check: C) -> Result<(), Report>
     where
         C: Service<R, Response = S>,
-        C::Error: Into<BoxError>,
+        C::Error: Into<Error>,
     {
         while let Some((req, expected_rsp)) = self.messages.next() {
             // These unwraps could propagate errors with the correct
             // bound on C::Error
-            let rsp = to_check
+            let fut = to_check
                 .ready_and()
                 .await
                 .map_err(Into::into)
-                .map_err(|e| eyre!(e))?
+                .map_err(|e| eyre!(e))
-                .call(req)
+                .expect("expected service to not fail during execution of transcript");
-                .await
+
-                .map_err(Into::into)
+            let response = fut.call(req).await;
-                .map_err(|e| eyre!(e))?;
+
-            ensure!(
+            match (response, expected_rsp) {
-                rsp == expected_rsp,
+                (Ok(rsp), Ok(expected_rsp)) => {
-                "Expected {:?}, got {:?}",
+                    if rsp != expected_rsp {
-                expected_rsp,
+                        Err(eyre!(
-                rsp
+                            "response doesn't match transcript's expected response"
-            );
+                        ))
+                        .with_section(|| format!("{:?}", expected_rsp).header("Expected Response:"))
+                        .with_section(|| format!("{:?}", rsp).header("Found Response:"))?;
+                    }
+                }
+                (Ok(rsp), Err(error_checker)) => {
+                    let error = Err(eyre!("received a response when an error was expected"))
+                        .with_section(|| format!("{:?}", rsp).header("Found Response:"));
+
+                    let error = match std::panic::catch_unwind(|| error_checker.mock()) {
+                        Ok(expected_err) => error.with_section(|| {
+                            format!("{:?}", expected_err).header("Expected Error:")
+                        }),
+                        Err(pi) => {
+                            let payload = pi
+                                .downcast_ref::<String>()
+                                .cloned()
+                                .or_else(|| pi.downcast_ref::<&str>().map(ToString::to_string))
+                                .unwrap_or_else(|| "<non string panic payload>".into());
+
+                            error
+                                .section(payload.header("Panic:"))
+                                .wrap_err("ErrorChecker panicked when producing expected response")
+                        }
+                    };
+
+                    error?;
+                }
+                (Err(e), Ok(expected_rsp)) => {
+                    Err(eyre!("received an error when a response was expected"))
+                        .with_error(|| ErrorCheckerError(e.into()))
+                        .with_section(|| {
+                            format!("{:?}", expected_rsp).header("Expected Response:")
+                        })?
+                }
+                (Err(e), Err(error_checker)) => {
+                    error_checker.check(e.into())?;
+                    continue;
+                }
+            }
         }
         Ok(())
     }
@@ -63,7 +143,7 @@ where
 impl<R, S, I> Service<R> for Transcript<R, S, I>
 where
     R: Debug + Eq,
-    I: Iterator<Item = (R, S)>,
+    I: Iterator<Item = (R, Result<S, TransError>)>,
 {
     type Response = S;
     type Error = Report;
@@ -75,14 +155,21 @@ where
 
     fn call(&mut self, request: R) -> Self::Future {
         if let Some((expected_request, response)) = self.messages.next() {
+            match response {
+                Ok(response) => {
                     if request == expected_request {
                         ready(Ok(response))
                     } else {
-                ready(Err(eyre!(
+                        ready(
-                    "Expected {:?}, got {:?}",
+                            Err(eyre!("received unexpected request"))
-                    expected_request,
+                                .with_section(|| {
-                    request
+                                    format!("{:?}", expected_request).header("Expected Request:")
-                )))
+                                })
+                                .with_section(|| format!("{:?}", request).header("Found Request:")),
+                        )
+                    }
+                }
+                Err(check_fn) => ready(Err(check_fn.mock())),
             }
         } else {
             ready(Err(eyre!("Got request after transcript ended")))

zebra-test/tests/transcript.rs

@@ -1,22 +1,26 @@
-use tower::{Service, ServiceExt};
+#![allow(clippy::try_err)]
 
+use tower::{Service, ServiceExt};
+use zebra_test::transcript::TransError;
 use zebra_test::transcript::Transcript;
 
-const TRANSCRIPT_DATA: [(&str, &str); 4] = [
+const TRANSCRIPT_DATA: [(&str, Result<&str, TransError>); 4] = [
-    ("req1", "rsp1"),
+    ("req1", Ok("rsp1")),
-    ("req2", "rsp2"),
+    ("req2", Ok("rsp2")),
-    ("req3", "rsp3"),
+    ("req3", Ok("rsp3")),
-    ("req4", "rsp4"),
+    ("req4", Ok("rsp4")),
 ];
 
 #[tokio::test]
 async fn transcript_returns_responses_and_ends() {
+    zebra_test::init();
+
     let mut svc = Transcript::from(TRANSCRIPT_DATA.iter().cloned());
 
     for (req, rsp) in TRANSCRIPT_DATA.iter() {
         assert_eq!(
             svc.ready_and().await.unwrap().call(req).await.unwrap(),
-            *rsp,
+            *rsp.as_ref().unwrap()
         );
     }
     assert!(svc.ready_and().await.unwrap().call("end").await.is_err());
@@ -24,6 +28,8 @@ async fn transcript_returns_responses_and_ends() {
 
 #[tokio::test]
 async fn transcript_errors_wrong_request() {
+    zebra_test::init();
+
     let mut svc = Transcript::from(TRANSCRIPT_DATA.iter().cloned());
 
     assert_eq!(
@@ -35,7 +41,31 @@ async fn transcript_errors_wrong_request() {
 
 #[tokio::test]
 async fn self_check() {
+    zebra_test::init();
+
     let t1 = Transcript::from(TRANSCRIPT_DATA.iter().cloned());
     let t2 = Transcript::from(TRANSCRIPT_DATA.iter().cloned());
     assert!(t1.check(t2).await.is_ok());
 }
+
+#[derive(Debug, thiserror::Error)]
+#[error("Error")]
+struct Error;
+
+const TRANSCRIPT_DATA2: [(&str, Result<&str, TransError>); 4] = [
+    ("req1", Ok("rsp1")),
+    ("req2", Ok("rsp2")),
+    ("req3", Ok("rsp3")),
+    ("req4", Err(TransError::Any)),
+];
+
+#[tokio::test]
+async fn self_check_err() {
+    zebra_test::init();
+
+    let t1 = Transcript::from(TRANSCRIPT_DATA2.iter().cloned());
+    let t2 = Transcript::from(TRANSCRIPT_DATA2.iter().cloned());
+    t1.check(t2)
+        .await
+        .expect("transcript acting as the mocker and verifier should always pass")
+}