change(ci): add acceptance test for getblocktemplate RPC in CI, and fix RPC bugs (#5761)

* Re-apply: add acceptance test for getblocktemplate method in CI (#5653)

Revert "Revert "change(tests): add acceptance test for getblocktemplate method in CI (#5653)" (#5672)"

This reverts commit 6446e0ec1b.

* Fix incorrect MAX_CONTEXT_BLOCKS assertion in state

* Actually negate the miner fee for the RPC output

* Try the RPC again after waiting for transactions to verify

* Log before the test waits for the mempool to verify transactions

* Use the new ssh key secrets in CI

.github/workflows/continous-integration-docker.patch.yml

@@ -90,6 +90,12 @@ jobs:
     steps:
       - run: 'echo "No build required"'
 
+  get-block-template-test:
+    name: get block template / Run get-block-template test
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
   submit-block-test:
     name: submit block / Run submit-block test
     runs-on: ubuntu-latest

.github/workflows/continous-integration-docker.yml

@@ -593,6 +593,32 @@ jobs:
       lwd_state_dir: 'lwd-cache'
     secrets: inherit
 
+  # Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state
+  #
+  # Runs:
+  # - after every PR is merged to `main`
+  # - on every PR update
+  #
+  # If the state version has changed, waits for the new cached states to be created.
+  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
+  get-block-template-test:
+    name: get block template
+    needs: test-full-sync
+    uses: ./.github/workflows/deploy-gcp-tests.yml
+    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }}
+    with:
+      app_name: zebrad
+      test_id: get-block-template
+      test_description: Test getblocktemplate RPC method via Zebra's rpc server
+      test_variables: '-e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
+      needs_zebra_state: true
+      needs_lwd_state: false
+      saves_to_disk: false
+      disk_suffix: tip
+      root_state_path: '/var/cache'
+      zebra_state_dir: 'zebrad-cache'
+    secrets: inherit
+
   # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state
   #
   # Runs:

docker/entrypoint.sh

@@ -78,6 +78,10 @@ case "$1" in
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
             ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory")
             cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd
+        elif [[ "$TEST_GET_BLOCK_TEMPLATE" -eq "1" ]]; then
+            # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server.
+            ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")
+            cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template
         elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then
             # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port.
             ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR  "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory")

zebra-chain/src/amount.rs

@@ -398,6 +398,8 @@ where
     }
 }
 
+// TODO: add infalliable impls for NonNegative <-> NegativeOrZero,
+//       when Rust uses trait output types to disambiguate overlapping impls.
 impl<C> std::ops::Neg for Amount<C>
 where
     C: Constraint,

zebra-rpc/src/methods/get_block_template_rpcs/types/transaction.rs

@@ -100,7 +100,7 @@ impl TransactionTemplate<NegativeOrZero> {
              must have exactly one input, which must be a coinbase input",
         );
 
-        let miner_fee = miner_fee
+        let miner_fee = (-miner_fee)
             .constrain()
             .expect("negating a NonNegative amount always results in a valid NegativeOrZero");
 

zebra-state/src/service/read/difficulty.rs

@@ -68,7 +68,11 @@ where
 
     // The getblocktemplate RPC returns an error if Zebra is not synced to the tip.
     // So this will never happen in production code.
-    assert!(relevant_data.len() < MAX_CONTEXT_BLOCKS);
+    assert_eq!(
+        relevant_data.len(),
+        MAX_CONTEXT_BLOCKS,
+        "getblocktemplate RPC called with a near-empty state: should have returned an error",
+    );
 
     let current_system_time = chrono::Utc::now();
 

zebrad/tests/acceptance.rs

@@ -103,6 +103,12 @@
 //!
 //! ## Getblocktemplate tests
 //!
+//! Example of how to run the get_block_template test:
+//!
+//! ```console
+//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/chain cargo test get_block_template --features getblocktemplate-rpcs --release  -- --ignored --nocapture
+//! ```
+//!
 //! Example of how to run the submit_block test:
 //!
 //! ```console
@@ -2154,9 +2160,19 @@ async fn lightwalletd_wallet_grpc_tests() -> Result<()> {
     common::lightwalletd::wallet_grpc_test::run().await
 }
 
+/// Test successful getblocktemplate rpc call
+///
+/// See [`common::get_block_template_rpcs::get_block_template`] for more information.
+#[tokio::test]
+#[ignore]
+#[cfg(feature = "getblocktemplate-rpcs")]
+async fn get_block_template() -> Result<()> {
+    common::get_block_template_rpcs::get_block_template::run().await
+}
+
 /// Test successful submitblock rpc call
 ///
-/// See [`common::getblocktemplate`] for more information.
+/// See [`common::get_block_template_rpcs::submit_block`] for more information.
 #[tokio::test]
 #[ignore]
 #[cfg(feature = "getblocktemplate-rpcs")]

zebrad/tests/common/get_block_template_rpcs.rs

@@ -1,3 +1,4 @@
 //! Acceptance tests for getblocktemplate RPC methods in Zebra.
 
+pub(crate) mod get_block_template;
 pub(crate) mod submit_block;

zebrad/tests/common/get_block_template_rpcs/get_block_template.rs

@@ -0,0 +1,114 @@
+//! Test getblocktemplate RPC method.
+//!
+//! This test requires a cached chain state that is partially synchronized close to the
+//! network chain tip height. It will finish the sync and update the cached chain state.
+//!
+//! After finishing the sync, it will call getblocktemplate.
+
+use std::time::Duration;
+
+use color_eyre::eyre::{eyre, Context, Result};
+
+use zebra_chain::parameters::Network;
+
+use crate::common::{
+    launch::{can_spawn_zebrad_for_rpc, spawn_zebrad_for_rpc},
+    rpc_client::RPCRequestClient,
+    sync::{check_sync_logs_until, MempoolBehavior, SYNC_FINISHED_REGEX},
+    test_type::TestType,
+};
+
+/// How long the test waits for the mempool to download and verify transactions.
+///
+/// We've seen it take anywhere from 1-45 seconds for the mempool to have some transactions in it.
+pub const EXPECTED_MEMPOOL_TRANSACTION_TIME: Duration = Duration::from_secs(45);
+
+/// Launch Zebra, wait for it to sync, and check the getblocktemplate RPC returns without errors.
+pub(crate) async fn run() -> Result<()> {
+    let _init_guard = zebra_test::init();
+
+    // We want a zebra state dir in place,
+    let test_type = TestType::UpdateZebraCachedStateWithRpc;
+    let test_name = "get_block_template_test";
+    let network = Network::Mainnet;
+
+    // Skip the test unless the user specifically asked for it and there is a zebrad_state_path
+    if !can_spawn_zebrad_for_rpc(test_name, test_type) {
+        return Ok(());
+    }
+
+    tracing::info!(
+        ?network,
+        ?test_type,
+        "running getblocktemplate test using zebrad",
+    );
+
+    let should_sync = true;
+    let (zebrad, zebra_rpc_address) =
+        spawn_zebrad_for_rpc(network, test_name, test_type, should_sync)?
+            .ok_or_else(|| eyre!("getblocktemplate test requires a cached state"))?;
+
+    let rpc_address = zebra_rpc_address.expect("test type must have RPC port");
+
+    let mut zebrad = check_sync_logs_until(
+        zebrad,
+        network,
+        SYNC_FINISHED_REGEX,
+        MempoolBehavior::ShouldAutomaticallyActivate,
+        true,
+    )?;
+
+    tracing::info!(
+        "calling getblocktemplate RPC method at {rpc_address}, \
+         with a mempool that is likely empty...",
+    );
+    let getblocktemplate_response = RPCRequestClient::new(rpc_address)
+        .call("getblocktemplate", "[]".to_string())
+        .await?;
+
+    let is_response_success = getblocktemplate_response.status().is_success();
+    let response_text = getblocktemplate_response.text().await?;
+
+    tracing::info!(
+        response_text,
+        "got getblocktemplate response, might not have transactions"
+    );
+
+    assert!(is_response_success);
+
+    tracing::info!(
+        "waiting {EXPECTED_MEMPOOL_TRANSACTION_TIME:?} for the mempool \
+         to download and verify some transactions...",
+    );
+    tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await;
+
+    tracing::info!(
+        "calling getblocktemplate RPC method at {rpc_address}, \
+         with a mempool that likely has transactions...",
+    );
+    let getblocktemplate_response = RPCRequestClient::new(rpc_address)
+        .call("getblocktemplate", "[]".to_string())
+        .await?;
+
+    let is_response_success = getblocktemplate_response.status().is_success();
+    let response_text = getblocktemplate_response.text().await?;
+
+    tracing::info!(
+        response_text,
+        "got getblocktemplate response, hopefully with transactions"
+    );
+
+    assert!(is_response_success);
+
+    zebrad.kill(false)?;
+
+    let output = zebrad.wait_with_output()?;
+    let output = output.assert_failure()?;
+
+    // [Note on port conflict](#Note on port conflict)
+    output
+        .assert_was_killed()
+        .wrap_err("Possible port conflict. Are there other acceptance tests running?")?;
+
+    Ok(())
+}

zebrad/tests/common/get_block_template_rpcs/submit_block.rs

@@ -22,7 +22,6 @@ use crate::common::{
 /// Number of blocks past the finalized to retrieve and submit.
 const MAX_NUM_FUTURE_BLOCKS: u32 = 3;
 
-#[allow(clippy::print_stderr)]
 pub(crate) async fn run() -> Result<()> {
     let _init_guard = zebra_test::init();
 

zebrad/tests/common/test_type.rs

@@ -176,6 +176,11 @@ impl TestType {
             return Some(Ok(config));
         }
 
+        #[cfg(feature = "getblocktemplate-rpcs")]
+        let _ = config.mining.miner_address.insert(
+            zebra_chain::transparent::Address::from_script_hash(config.network.network, [0x7e; 20]),
+        );
+
         let zebra_state_path = self.zebrad_state_path(test_name)?;
 
         config.sync.checkpoint_verify_concurrency_limit =