fix(test): avoid failing getblocktemplate acceptance test when the chain tip changes (#6091)

* adds basic usage of long polling in gbt test

* adds !submit_old check before cancelling proposing a new template

* Removes break statement in long polling task

* Update zebrad/tests/common/rpc_client.rs

Co-authored-by: teor <teor@riseup.net>

* use blocking_send and watch channel

* fix "cannot block the current thread from within a runtime"

* Reduces interval between proposals and increases num proposals required.

* Runs rate-limiting sleeps in parallel to validation

* corrects comment.

---------

Co-authored-by: teor <teor@riseup.net>

zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs

@@ -3,7 +3,7 @@
 use crate::methods::get_block_template_rpcs::types::{hex_data::HexData, long_poll::LongPollId};
 
 /// Defines whether the RPC method should generate a block template or attempt to validate a block proposal.
-#[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)]
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub enum GetBlockTemplateRequestMode {
     /// Indicates a request for a block template.
@@ -20,7 +20,7 @@ impl Default for GetBlockTemplateRequestMode {
 }
 
 /// Valid `capabilities` values that indicate client-side support.
-#[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)]
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub enum GetBlockTemplateCapability {
     /// Long Polling support.
@@ -59,7 +59,7 @@ pub enum GetBlockTemplateCapability {
 ///
 /// The `data` field must be provided in `proposal` mode, and must be omitted in `template` mode.
 /// All other fields are optional.
-#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, Default)]
+#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize, Default)]
 pub struct JsonParameters {
     /// Defines whether the RPC method should generate a block template or attempt to
     /// validate block data, checking against all of the server's usual acceptance rules

zebra-rpc/src/methods/get_block_template_rpcs/types/hex_data.rs

@@ -2,5 +2,5 @@
 //! for the `submitblock` RPC method.
 
 /// Deserialize hex-encoded strings to bytes.
-#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub struct HexData(#[serde(with = "hex")] pub Vec<u8>);

zebrad/tests/common/get_block_template_rpcs/get_block_template.rs

@@ -9,9 +9,12 @@ use std::time::Duration;
 
 use color_eyre::eyre::{eyre, Context, Result};
 
+use futures::FutureExt;
 use zebra_chain::{parameters::Network, serialization::ZcashSerialize};
 use zebra_rpc::methods::get_block_template_rpcs::{
-    get_block_template::{proposal::TimeSource, ProposalResponse},
+    get_block_template::{
+        proposal::TimeSource, GetBlockTemplate, JsonParameters, ProposalResponse,
+    },
     types::get_block_template::proposal_block_from_template,
 };
 
@@ -27,9 +30,18 @@ use crate::common::{
 /// We've seen it take anywhere from 1-45 seconds for the mempool to have some transactions in it.
 pub const EXPECTED_MEMPOOL_TRANSACTION_TIME: Duration = Duration::from_secs(45);
 
+/// Delay between getting block proposal results and cancelling long poll requests.
+pub const EXTRA_LONGPOLL_WAIT_TIME: Duration = Duration::from_millis(150);
+
+/// Delay between attempts to validate a template as block proposals.
+///
+/// Running many iterations in short intervals tests that long poll requests correctly
+/// return `submit_old: false` when the old template becomes invalid.
+pub const BLOCK_PROPOSAL_INTERVAL: Duration = Duration::from_millis(300);
+
 /// Number of times we want to try submitting a block template as a block proposal at an interval
 /// that allows testing the varying mempool contents.
-const NUM_SUCCESSFUL_BLOCK_PROPOSALS_REQUIRED: usize = 10;
+const NUM_SUCCESSFUL_BLOCK_PROPOSALS_REQUIRED: usize = 1000;
 
 /// Launch Zebra, wait for it to sync, and check the getblocktemplate RPC returns without errors.
 pub(crate) async fn run() -> Result<()> {
@@ -90,13 +102,15 @@ pub(crate) async fn run() -> Result<()> {
 
     assert!(is_response_success);
 
+    tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await;
+
     for _ in 0..NUM_SUCCESSFUL_BLOCK_PROPOSALS_REQUIRED {
         tracing::info!(
             "waiting {EXPECTED_MEMPOOL_TRANSACTION_TIME:?} for the mempool \
              to download and verify some transactions...",
         );
 
-        tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await;
+        tokio::time::sleep(BLOCK_PROPOSAL_INTERVAL).await;
 
         tracing::info!(
             "calling getblocktemplate RPC method at {rpc_address}, \
@@ -104,9 +118,13 @@ pub(crate) async fn run() -> Result<()> {
                  to validate response result as a block proposal",
         );
 
-        try_validate_block_template(&client)
+        let (validation_result, _) = futures::future::join(
-            .await
+            try_validate_block_template(&client),
-            .expect("block proposal validation failed");
+            tokio::time::sleep(BLOCK_PROPOSAL_INTERVAL),
+        )
+        .await;
+
+        validation_result.expect("block proposal validation failed");
     }
 
     zebrad.kill(false)?;
@@ -134,47 +152,124 @@ pub(crate) async fn run() -> Result<()> {
 /// If the response result cannot be deserialized to `GetBlockTemplate` in 'template' mode
 /// or `ProposalResponse` in 'proposal' mode.
 async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> {
-    let response_json_result = client
+    let mut response_json_result: GetBlockTemplate = client
         .json_result_from_call("getblocktemplate", "[]".to_string())
         .await
-        .expect("response should be success output with with a serialized `GetBlockTemplate`");
+        .expect("response should be success output with a serialized `GetBlockTemplate`");
 
-    tracing::info!(
+    let (long_poll_result_tx, mut long_poll_result_rx) =
-        ?response_json_result,
+        tokio::sync::watch::channel(response_json_result.clone());
-        "got getblocktemplate response, hopefully with transactions"
+    let (done_tx, mut done_rx) = tokio::sync::mpsc::channel(1);
-    );
 
-    for time_source in TimeSource::valid_sources() {
+    {
-        // Propose a new block with an empty solution and nonce field
+        let client = client.clone();
+        let mut long_poll_id = response_json_result.long_poll_id.clone();
+
+        tokio::spawn(async move {
+            loop {
+                let long_poll_request = async {
+                    let long_poll_json_params = serde_json::to_string(&vec![JsonParameters {
+                        long_poll_id: Some(long_poll_id),
+                        ..Default::default()
+                    }])
+                    .expect("JsonParameters should serialize successfully");
+
+                    let result: GetBlockTemplate = client
+                            .json_result_from_call("getblocktemplate", long_poll_json_params)
+                            .await
+                            .expect(
+                                "response should be success output with a serialized `GetBlockTemplate`",
+                            );
+
+                    result
+                };
+
+                tokio::select! {
+                    _ = done_rx.recv() => {
+                        break;
+                    }
+
+                    long_poll_result = long_poll_request => {
+                        long_poll_id = long_poll_result.long_poll_id.clone();
+
+                        if let Some(false) = long_poll_result.submit_old {
+                            let _ = long_poll_result_tx.send(long_poll_result);
+                        }
+                    }
+                }
+            }
+        });
+    };
+
+    loop {
         tracing::info!(
-            "calling getblocktemplate with a block proposal and time source {time_source:?}...",
+            ?response_json_result,
+            "got getblocktemplate response, hopefully with transactions"
         );
 
-        let raw_proposal_block = hex::encode(
+        let mut proposal_requests = vec![];
-            proposal_block_from_template(&response_json_result, time_source)?
-                .zcash_serialize_to_vec()?,
-        );
 
-        let json_result = client
+        for time_source in TimeSource::valid_sources() {
-            .json_result_from_call(
+            // Propose a new block with an empty solution and nonce field
-                "getblocktemplate",
+            tracing::info!(
-                format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#),
+                "calling getblocktemplate with a block proposal and time source {time_source:?}...",
-            )
+            );
-            .await
-            .expect("response should be success output with with a serialized `ProposalResponse`");
 
-        tracing::info!(
+            let raw_proposal_block = hex::encode(
-            ?json_result,
+                proposal_block_from_template(&response_json_result, time_source)?
-            ?time_source,
+                    .zcash_serialize_to_vec()?,
-            "got getblocktemplate proposal response"
+            );
-        );
 
-        if let ProposalResponse::Rejected(reject_reason) = json_result {
+            proposal_requests.push(async move {
-            Err(eyre!(
+                (
-                "unsuccessful block proposal validation, reason: {reject_reason:?}"
+                    client
-            ))?;
+                        .json_result_from_call(
-        } else {
+                            "getblocktemplate",
-            assert_eq!(ProposalResponse::Valid, json_result);
+                            format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#),
+                        )
+                        .await,
+                    time_source,
+                )
+            });
+        }
+
+        tokio::select! {
+            Ok(()) = long_poll_result_rx.changed() => {
+                tracing::info!("got longpolling response with submitold of false before result of proposal tests");
+
+                // The task that handles the long polling request will keep checking for
+                // a new template with `submit_old`: false
+                response_json_result = long_poll_result_rx.borrow().clone();
+
+                continue;
+            },
+
+            proposal_results = futures::future::join_all(proposal_requests).then(|results| async move {
+                tokio::time::sleep(EXTRA_LONGPOLL_WAIT_TIME).await;
+                results
+            }) => {
+                let _ = done_tx.send(()).await;
+                for (proposal_result, time_source) in proposal_results {
+                    let proposal_result = proposal_result
+                        .expect("response should be success output with with a serialized `ProposalResponse`");
+
+                    tracing::info!(
+                        ?proposal_result,
+                        ?time_source,
+                        "got getblocktemplate proposal response"
+                    );
+
+                    if let ProposalResponse::Rejected(reject_reason) = proposal_result {
+                        Err(eyre!(
+                            "unsuccessful block proposal validation, reason: {reject_reason:?}"
+                        ))?;
+                    } else {
+                        assert_eq!(ProposalResponse::Valid, proposal_result);
+                    }
+                }
+
+                break;
+            }
         }
     }
 

zebrad/tests/common/rpc_client.rs

@@ -8,6 +8,7 @@ use reqwest::Client;
 use color_eyre::{eyre::eyre, Result};
 
 /// An http client for making Json-RPC requests
+#[derive(Clone, Debug)]
 pub struct RPCRequestClient {
     client: Client,
     rpc_address: SocketAddr,