From 20eeddcaab9a62433775d0ba1a81f7389fd7cd00 Mon Sep 17 00:00:00 2001 From: Janito Vaqueiro Ferreira Filho Date: Tue, 6 Jul 2021 22:06:11 -0300 Subject: [PATCH] Parse `MSG_WTX` inventory type (part of ZIP-239) (#2446) * Rename constant to `MIN_INVENTORY_HASH_SIZE` Because the size is not constant anymore, since the `MSG_WTX` inventory type is larger. * Add `InventoryHash::smallest_types_strategy` A method for a proptest strategy that generates the `InventoryHash` variants that have the smallest serialized size. * Update proptest to use only smallest inventories In order to properly test the maximum allocation. * Add intra-doc links in some method documentation Make it easier to navigate from the documentation of the proptest strategies to the variants they generate. * Parse `MSG_WTX` inventory type Avoid returning an error if a received `GetData` or `Inv` message contains a `MSG_WTX` inventory type. This prevents Zebra from disconnecting from peers that announce V5 transactions. * Fix inventory hash size proptest The serialized size now depends on what type of `InventoryHash` is being tested. * Implement serialization of `InventoryHash::Wtx` For now it just copies the stored bytes, in order to allow the tests to run correctly. * Test if `MSG_WTX` inventory is parsed correctly Create some mock input bytes representing a serialized `MSG_WTX` inventory item, and check that it can be deserialized successfully. * Generate arbitrary `InventoryHash::Wtx` for tests Create a strategy that only generates `InventoryHash::Wtx` instances, and also update the `Arbitrary` implementation for `InventoryHash` to also generate `Wtx` variants. * Test `InventoryHash` serialization roundtrip Given an arbitrary `InventoryHash`, check that it does not change after being serialized and deserialized. Currently, `InventoryHash::Wtx` can't be serialized, so this test will is expected to panic for now, but it will fail once the serialization code is implemented, and then the `should_panic` should be removed. * Test deserialize `InventoryHash` from random bytes Create an random input vector of bytes, and try to deserialize an `InventoryHash` from it. This should either succeed or fail in an expected way. * Remove redundant attribute The attribute is redundant because the `arbitrary` module already has that attribute. * Implement `Message::inv_strategy()` A method to return a proptest strategy that creates `Message::Inv` instances. * Implement `Message::get_data_strategy()` A method that returns a proptest strategy that creates `Message::GetData` instances. * Test encode/decode roundtrip of some `Message`s Create a `Message` instance, encode it and then decode it using a `Codec` instance and check that the result is the same as the initial `Message`. For now, this only tests `Message::Inv` and `Message::GetData`, because these are the variants that are related to the scope of the current set of changes to support parsing the `MSG_WTX` inventory type. Even so, the test relies on being able to serialize an `InventoryHash::Wtx`, which is currently not implemented. Therefore the test was marked as `should_panic` until the serialization code is implemented. --- .../protocol/external/tests/prop.txt | 7 ++ .../src/protocol/external/arbitrary.rs | 53 +++++++++-- zebra-network/src/protocol/external/inv.rs | 42 ++++++--- zebra-network/src/protocol/external/tests.rs | 2 + .../protocol/external/tests/preallocate.rs | 21 +++-- .../src/protocol/external/tests/prop.rs | 92 +++++++++++++++++++ .../src/protocol/external/tests/vectors.rs | 26 ++++++ 7 files changed, 219 insertions(+), 24 deletions(-) create mode 100644 zebra-network/proptest-regressions/protocol/external/tests/prop.txt create mode 100644 zebra-network/src/protocol/external/tests/prop.rs create mode 100644 zebra-network/src/protocol/external/tests/vectors.rs diff --git a/zebra-network/proptest-regressions/protocol/external/tests/prop.txt b/zebra-network/proptest-regressions/protocol/external/tests/prop.txt new file mode 100644 index 00000000..3e8e4e29 --- /dev/null +++ b/zebra-network/proptest-regressions/protocol/external/tests/prop.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 62951c0bf7f003f29184881befbd1e8144493b3b14a6dd738ecef9e4c8c06148 # shrinks to inventory_hash = Wtx diff --git a/zebra-network/src/protocol/external/arbitrary.rs b/zebra-network/src/protocol/external/arbitrary.rs index e736f409..4a20e34a 100644 --- a/zebra-network/src/protocol/external/arbitrary.rs +++ b/zebra-network/src/protocol/external/arbitrary.rs @@ -1,16 +1,18 @@ -use proptest::{arbitrary::any, arbitrary::Arbitrary, prelude::*}; +use std::convert::TryInto; -use super::{types::PeerServices, InventoryHash}; +use proptest::{arbitrary::any, arbitrary::Arbitrary, collection::vec, prelude::*}; + +use super::{types::PeerServices, InventoryHash, Message}; use zebra_chain::{block, transaction}; impl InventoryHash { - /// Generate a proptest strategy for Inv Errors + /// Generate a proptest strategy for [`InventoryHash::Error`]s. pub fn error_strategy() -> BoxedStrategy { Just(InventoryHash::Error).boxed() } - /// Generate a proptest strategy for Inv Tx hashes + /// Generate a proptest strategy for [`InventoryHash::Tx`] hashes. pub fn tx_strategy() -> BoxedStrategy { // using any:: causes a trait impl error // when building the zebra-network crate separately @@ -20,7 +22,7 @@ impl InventoryHash { .boxed() } - /// Generate a proptest strategy for Inv Block hashes + /// Generate a proptest strategy for [`InventotryHash::Block`] hashes. pub fn block_strategy() -> BoxedStrategy { (any::<[u8; 32]>()) .prop_map(block::Hash) @@ -28,13 +30,36 @@ impl InventoryHash { .boxed() } - /// Generate a proptest strategy for Inv FilteredBlock hashes + /// Generate a proptest strategy for [`InventoryHash::FilteredBlock`] hashes. pub fn filtered_block_strategy() -> BoxedStrategy { (any::<[u8; 32]>()) .prop_map(block::Hash) .prop_map(InventoryHash::FilteredBlock) .boxed() } + + /// Generate a proptest strategy for [`InventoryHash::Wtx`] hashes. + pub fn wtx_strategy() -> BoxedStrategy { + vec(any::(), 64) + .prop_map(|bytes| InventoryHash::Wtx(bytes.try_into().unwrap())) + .boxed() + } + + /// Generate a proptest strategy for [`InventoryHash`] variants of the smallest serialized size. + pub fn smallest_types_strategy() -> BoxedStrategy { + InventoryHash::arbitrary() + .prop_filter( + "inventory type is not one of the smallest", + |inventory_hash| match inventory_hash { + InventoryHash::Error + | InventoryHash::Tx(_) + | InventoryHash::Block(_) + | InventoryHash::FilteredBlock(_) => true, + InventoryHash::Wtx(_) => false, + }, + ) + .boxed() + } } impl Arbitrary for InventoryHash { @@ -46,6 +71,7 @@ impl Arbitrary for InventoryHash { Self::tx_strategy(), Self::block_strategy(), Self::filtered_block_strategy(), + Self::wtx_strategy(), ] .boxed() } @@ -53,7 +79,6 @@ impl Arbitrary for InventoryHash { type Strategy = BoxedStrategy; } -#[cfg(any(test, feature = "proptest-impl"))] impl Arbitrary for PeerServices { type Parameters = (); @@ -65,3 +90,17 @@ impl Arbitrary for PeerServices { type Strategy = BoxedStrategy; } + +impl Message { + /// Create a strategy that only generates [`Message::Inv`] messages. + pub fn inv_strategy() -> BoxedStrategy { + any::>().prop_map(Message::Inv).boxed() + } + + /// Create a strategy that only generates [`Message::GetData`] messages. + pub fn get_data_strategy() -> BoxedStrategy { + any::>() + .prop_map(Message::GetData) + .boxed() + } +} diff --git a/zebra-network/src/protocol/external/inv.rs b/zebra-network/src/protocol/external/inv.rs index 9eb3e3c4..44b7df86 100644 --- a/zebra-network/src/protocol/external/inv.rs +++ b/zebra-network/src/protocol/external/inv.rs @@ -38,6 +38,15 @@ pub enum InventoryHash { /// rather than a block message; this only works if a bloom filter has been /// set. FilteredBlock(block::Hash), + /// A pair with the hash of a V5 transaction and the [Authorizing Data Commitment][auth_digest]. + /// + /// Introduced by [ZIP-239][zip239], which is analogous to Bitcoin's [BIP-339][bip339]. + /// + /// [auth_digest]: https://zips.z.cash/zip-0244#authorizing-data-commitment + /// [zip239]: https://zips.z.cash/zip-0239 + /// [bip339]: https://github.com/bitcoin/bips/blob/master/bip-0339.mediawiki + // TODO: Actually handle this variant once the mempool is implemented + Wtx([u8; 64]), } impl From for InventoryHash { @@ -56,14 +65,15 @@ impl From for InventoryHash { impl ZcashSerialize for InventoryHash { fn zcash_serialize(&self, mut writer: W) -> Result<(), std::io::Error> { - let (code, bytes) = match *self { - InventoryHash::Error => (0, [0; 32]), - InventoryHash::Tx(hash) => (1, hash.0), - InventoryHash::Block(hash) => (2, hash.0), - InventoryHash::FilteredBlock(hash) => (3, hash.0), + let (code, bytes): (_, &[u8]) = match self { + InventoryHash::Error => (0, &[0; 32]), + InventoryHash::Tx(hash) => (1, &hash.0), + InventoryHash::Block(hash) => (2, &hash.0), + InventoryHash::FilteredBlock(hash) => (3, &hash.0), + InventoryHash::Wtx(bytes) => (5, bytes), }; writer.write_u32::(code)?; - writer.write_all(&bytes)?; + writer.write_all(bytes)?; Ok(()) } } @@ -77,18 +87,28 @@ impl ZcashDeserialize for InventoryHash { 1 => Ok(InventoryHash::Tx(transaction::Hash(bytes))), 2 => Ok(InventoryHash::Block(block::Hash(bytes))), 3 => Ok(InventoryHash::FilteredBlock(block::Hash(bytes))), + 5 => { + let auth_digest = reader.read_32_bytes()?; + + let mut wtx_bytes = [0u8; 64]; + wtx_bytes[..32].copy_from_slice(&bytes); + wtx_bytes[32..].copy_from_slice(&auth_digest); + + Ok(InventoryHash::Wtx(wtx_bytes)) + } _ => Err(SerializationError::Parse("invalid inventory code")), } } } -/// The serialized size of an [`InventoryHash`]. -pub(crate) const INV_HASH_SIZE: usize = 36; +/// The minimum serialized size of an [`InventoryHash`]. +pub(crate) const MIN_INV_HASH_SIZE: usize = 36; impl TrustedPreallocate for InventoryHash { fn max_allocation() -> u64 { - // An Inventory hash takes 36 bytes, and we reserve at least one byte for the Vector length - // so we can never receive more than ((MAX_PROTOCOL_MESSAGE_LEN - 1) / 36) in a single message - ((MAX_PROTOCOL_MESSAGE_LEN - 1) / INV_HASH_SIZE) as u64 + // An Inventory hash takes at least 36 bytes, and we reserve at least one byte for the + // Vector length so we can never receive more than ((MAX_PROTOCOL_MESSAGE_LEN - 1) / 36) in + // a single message + ((MAX_PROTOCOL_MESSAGE_LEN - 1) / MIN_INV_HASH_SIZE) as u64 } } diff --git a/zebra-network/src/protocol/external/tests.rs b/zebra-network/src/protocol/external/tests.rs index 67af9b07..3242b19e 100644 --- a/zebra-network/src/protocol/external/tests.rs +++ b/zebra-network/src/protocol/external/tests.rs @@ -1 +1,3 @@ mod preallocate; +mod prop; +mod vectors; diff --git a/zebra-network/src/protocol/external/tests/preallocate.rs b/zebra-network/src/protocol/external/tests/preallocate.rs index 21ef38bb..77709364 100644 --- a/zebra-network/src/protocol/external/tests/preallocate.rs +++ b/zebra-network/src/protocol/external/tests/preallocate.rs @@ -1,6 +1,6 @@ //! Tests for trusted preallocation during deserialization. -use super::super::inv::{InventoryHash, INV_HASH_SIZE}; +use super::super::inv::InventoryHash; use zebra_chain::serialization::{TrustedPreallocate, ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN}; @@ -8,21 +8,30 @@ use proptest::prelude::*; use std::convert::TryInto; proptest! { - /// Confirm that each InventoryHash takes exactly INV_HASH_SIZE bytes when serialized. - /// This verifies that our calculated [`TrustedPreallocate::max_allocation`] is indeed an upper bound. + /// Confirm that each InventoryHash takes the expected size in bytes when serialized. #[test] - fn inv_hash_size_is_correct(inv in InventoryHash::arbitrary()) { + fn inv_hash_size_is_correct(inv in any::()) { let serialized_inv = inv .zcash_serialize_to_vec() .expect("Serialization to vec must succeed"); - assert!(serialized_inv.len() == INV_HASH_SIZE); + + let expected_size = match inv { + InventoryHash::Error + | InventoryHash::Tx(_) + | InventoryHash::Block(_) + | InventoryHash::FilteredBlock(_) => 32 + 4, + + InventoryHash::Wtx(_) => 32 + 32 + 4, + }; + + assert_eq!(serialized_inv.len(), expected_size); } /// Verifies that... /// 1. The smallest disallowed vector of `InventoryHash`s is too large to fit in a legal Zcash message /// 2. The largest allowed vector is small enough to fit in a legal Zcash message #[test] - fn inv_hash_max_allocation_is_correct(inv in InventoryHash::arbitrary()) { + fn inv_hash_max_allocation_is_correct(inv in InventoryHash::smallest_types_strategy()) { let max_allocation: usize = InventoryHash::max_allocation().try_into().unwrap(); let mut smallest_disallowed_vec = Vec::with_capacity(max_allocation + 1); for _ in 0..(InventoryHash::max_allocation() + 1) { diff --git a/zebra-network/src/protocol/external/tests/prop.rs b/zebra-network/src/protocol/external/tests/prop.rs new file mode 100644 index 00000000..04c7a1f3 --- /dev/null +++ b/zebra-network/src/protocol/external/tests/prop.rs @@ -0,0 +1,92 @@ +use bytes::BytesMut; +use proptest::{collection::vec, prelude::*}; +use tokio_util::codec::{Decoder, Encoder}; + +use zebra_chain::serialization::{ + SerializationError, ZcashDeserializeInto, ZcashSerialize, MAX_PROTOCOL_MESSAGE_LEN, +}; + +use super::super::{Codec, InventoryHash, Message}; + +/// Maximum number of random input bytes to try to deserialize an [`InventoryHash`] from. +/// +/// This is two bytes larger than the maximum [`InventoryHash`] size. +const MAX_INVENTORY_HASH_BYTES: usize = 70; + +proptest! { + /// Test if [`InventoryHash`] is not changed after serializing and deserializing it. + #[test] + fn inventory_hash_roundtrip(inventory_hash in any::()) { + let mut bytes = Vec::new(); + + let serialization_result = inventory_hash.zcash_serialize(&mut bytes); + + prop_assert!(serialization_result.is_ok()); + prop_assert!(bytes.len() < MAX_INVENTORY_HASH_BYTES); + + let deserialized: Result = bytes.zcash_deserialize_into(); + + prop_assert!(deserialized.is_ok()); + prop_assert_eq!(deserialized.unwrap(), inventory_hash); + } + + /// Test attempting to deserialize an [`InventoryHash`] from random bytes. + #[test] + fn inventory_hash_from_random_bytes(input in vec(any::(), 0..MAX_INVENTORY_HASH_BYTES)) { + let deserialized: Result = input.zcash_deserialize_into(); + + if input.len() < 36 { + // Not enough bytes for any inventory hash + prop_assert!(deserialized.is_err()); + prop_assert_eq!( + deserialized.unwrap_err().to_string(), + "io error: failed to fill whole buffer" + ); + } else if input[1..4] != [0u8; 3] || input[0] > 5 || input[0] == 4 { + // Invalid inventory code + prop_assert!(matches!( + deserialized, + Err(SerializationError::Parse("invalid inventory code")) + )); + } else if input[0] == 5 && input.len() < 68 { + // Not enough bytes for a WTX inventory hash + prop_assert!(deserialized.is_err()); + prop_assert_eq!( + deserialized.unwrap_err().to_string(), + "io error: failed to fill whole buffer" + ); + } else { + // Deserialization should have succeeded + prop_assert!(deserialized.is_ok()); + + // Reserialize inventory hash + let mut bytes = Vec::new(); + let serialization_result = deserialized.unwrap().zcash_serialize(&mut bytes); + + prop_assert!(serialization_result.is_ok()); + + // Check that the reserialization produces the same bytes as the input + prop_assert!(bytes.len() <= input.len()); + prop_assert_eq!(&bytes, &input[..bytes.len()]); + } + } + + /// Test if a [`Message::{Inv, GetData}`] is not changed after encoding and decoding it. + // TODO: Update this test to cover all `Message` variants. + #[test] + fn inv_and_getdata_message_roundtrip( + message in prop_oneof!(Message::inv_strategy(), Message::get_data_strategy()), + ) { + let mut codec = Codec::builder().finish(); + let mut bytes = BytesMut::with_capacity(MAX_PROTOCOL_MESSAGE_LEN); + + let encoding_result = codec.encode(message.clone(), &mut bytes); + + prop_assert!(encoding_result.is_ok()); + + let decoded: Result, _> = codec.decode(&mut bytes); + + prop_assert!(decoded.is_ok()); + prop_assert_eq!(decoded.unwrap(), Some(message)); + } +} diff --git a/zebra-network/src/protocol/external/tests/vectors.rs b/zebra-network/src/protocol/external/tests/vectors.rs new file mode 100644 index 00000000..e4342d27 --- /dev/null +++ b/zebra-network/src/protocol/external/tests/vectors.rs @@ -0,0 +1,26 @@ +use std::io::Write; + +use byteorder::{LittleEndian, WriteBytesExt}; + +use zebra_chain::serialization::ZcashDeserializeInto; + +use super::super::InventoryHash; + +/// Test if deserializing [`InventoryHash::Wtx`] does not produce an error. +#[test] +fn parses_msg_wtx_inventory_type() { + let mut input = Vec::new(); + + input + .write_u32::(5) + .expect("Failed to write MSG_WTX code"); + input + .write_all(&[0u8; 64]) + .expect("Failed to write dummy inventory data"); + + let deserialized: InventoryHash = input + .zcash_deserialize_into() + .expect("Failed to deserialize dummy `InventoryHash::Wtx`"); + + assert_eq!(deserialized, InventoryHash::Wtx([0; 64])); +}