diff --git a/zebra-chain/src/transparent/address.rs b/zebra-chain/src/transparent/address.rs index e4a2ad1b..fda1bb28 100644 --- a/zebra-chain/src/transparent/address.rs +++ b/zebra-chain/src/transparent/address.rs @@ -6,15 +6,14 @@ use ripemd160::{Digest, Ripemd160}; use secp256k1::PublicKey; use sha2::Sha256; -#[cfg(test)] -use proptest::{arbitrary::Arbitrary, collection::vec, prelude::*}; - use crate::{ parameters::Network, serialization::{SerializationError, ZcashDeserialize, ZcashSerialize}, + transparent::Script, }; -use super::Script; +#[cfg(test)] +use proptest::prelude::*; /// Magic numbers used to identify what networks Transparent Addresses /// are associated with. @@ -41,7 +40,11 @@ mod magics { /// to a Bitcoin address just by removing the "t".) /// /// https://zips.z.cash/protocol/protocol.pdf#transparentaddrencoding -#[derive(Copy, Clone, Eq, PartialEq)] +#[derive(Copy, Clone, Eq, PartialEq, Hash)] +#[cfg_attr( + any(test, feature = "proptest-impl"), + derive(proptest_derive::Arbitrary) +)] pub enum Address { /// P2SH (Pay to Script Hash) addresses PayToScriptHash { @@ -208,6 +211,26 @@ impl Address { } } + /// Returns the network for this address. + pub fn network(&self) -> Network { + match *self { + Address::PayToScriptHash { network, .. } => network, + Address::PayToPublicKeyHash { network, .. } => network, + } + } + + /// Returns the hash bytes for this address, regardless of the address type. + /// + /// # Correctness + /// + /// Use [`ZcashSerialize`] and [`ZcashDeserialize`] for consensus-critical serialization. + pub fn hash_bytes(&self) -> [u8; 20] { + match *self { + Address::PayToScriptHash { script_hash, .. } => script_hash, + Address::PayToPublicKeyHash { pub_key_hash, .. } => pub_key_hash, + } + } + /// A hash of a transparent address payload, as used in /// transparent pay-to-script-hash and pay-to-publickey-hash /// addresses. @@ -224,46 +247,6 @@ impl Address { } } -#[cfg(test)] -impl Address { - fn p2pkh_strategy() -> impl Strategy { - (any::(), vec(any::(), 20)) - .prop_map(|(network, payload_bytes)| { - let mut bytes = [0; 20]; - bytes.copy_from_slice(payload_bytes.as_slice()); - Self::PayToPublicKeyHash { - network, - pub_key_hash: bytes, - } - }) - .boxed() - } - - fn p2sh_strategy() -> impl Strategy { - (any::(), vec(any::(), 20)) - .prop_map(|(network, payload_bytes)| { - let mut bytes = [0; 20]; - bytes.copy_from_slice(payload_bytes.as_slice()); - Self::PayToScriptHash { - network, - script_hash: bytes, - } - }) - .boxed() - } -} - -#[cfg(test)] -impl Arbitrary for Address { - type Parameters = (); - - fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy { - prop_oneof![Self::p2pkh_strategy(), Self::p2sh_strategy(),].boxed() - } - - type Strategy = BoxedStrategy; -} - #[cfg(test)] mod tests { diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index df8ba0e9..1168c7da 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -15,6 +15,7 @@ dirs = "4.0.0" displaydoc = "0.2.3" futures = "0.3.21" hex = "0.4.3" +itertools = "0.10.3" lazy_static = "1.4.0" metrics = "0.17.1" mset = "0.1.0" @@ -35,7 +36,6 @@ zebra-test = { path = "../zebra-test/", optional = true } [dev-dependencies] color-eyre = "0.6.0" -itertools = "0.10.3" once_cell = "1.10.0" spandoc = "0.2.1" diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs index 01e9b84b..583ccb2e 100644 --- a/zebra-state/src/constants.rs +++ b/zebra-state/src/constants.rs @@ -18,7 +18,7 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY; pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1; /// The database format version, incremented each time the database format changes. -pub const DATABASE_FORMAT_VERSION: u32 = 16; +pub const DATABASE_FORMAT_VERSION: u32 = 17; /// The maximum number of blocks to check for NU5 transactions, /// before we assume we are on a pre-NU5 legacy chain. diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index b1581587..d3bfc44f 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -68,10 +68,17 @@ pub struct DiskDb { /// /// [`rocksdb::WriteBatch`] is a batched set of database updates, /// which must be written to the database using `DiskDb::write(batch)`. +// +// TODO: move DiskDb, FinalizedBlock, and the source String into this struct, +// (DiskDb can be cloned), +// and make them accessible via read-only methods #[must_use = "batches must be written to the database"] pub struct DiskWriteBatch { /// The inner RocksDB write batch. batch: rocksdb::WriteBatch, + + /// The configured network. + network: Network, } /// Helper trait for inserting (Key, Value) pairs into rocksdb with a consistently @@ -298,11 +305,24 @@ impl ReadDisk for DiskDb { } impl DiskWriteBatch { - pub fn new() -> Self { + /// Creates and returns a new transactional batch write. + /// + /// # Correctness + /// + /// Each block must be written to the state inside a batch, so that: + /// - concurrent `ReadStateService` queries don't see half-written blocks, and + /// - if Zebra calls `exit`, panics, or crashes, half-written blocks are rolled back. + pub fn new(network: Network) -> Self { DiskWriteBatch { batch: rocksdb::WriteBatch::default(), + network, } } + + /// Returns the configured network for this write batch. + pub fn network(&self) -> Network { + self.network + } } impl DiskDb { @@ -344,7 +364,13 @@ impl DiskDb { // TODO: rename to tx_loc_by_hash (#3151) rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()), // Transparent + rocksdb::ColumnFamilyDescriptor::new("balance_by_transparent_addr", db_options.clone()), + // TODO: #3954 + //rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()), + // TODO: rename to utxo_by_out_loc (#3953) rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()), + // TODO: #3952 + //rocksdb::ColumnFamilyDescriptor::new("utxo_by_transparent_addr_loc", db_options.clone()), // Sprout rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()), rocksdb::ColumnFamilyDescriptor::new("sprout_anchors", db_options.clone()), diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs b/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs index 123a83d1..7a839d8c 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests/prop.rs @@ -3,17 +3,26 @@ use proptest::{arbitrary::any, prelude::*}; use zebra_chain::{ - amount::NonNegative, + amount::{Amount, NonNegative}, block::{self, Height}, + orchard, sapling, sprout, + transaction::{self, Transaction}, transparent, value_balance::ValueBalance, }; use crate::service::finalized_state::{ arbitrary::assert_value_properties, - disk_format::{block::MAX_ON_DISK_HEIGHT, transparent::OutputLocation, TransactionLocation}, + disk_format::{ + block::MAX_ON_DISK_HEIGHT, + transparent::{AddressBalanceLocation, AddressLocation, OutputLocation}, + IntoDisk, TransactionLocation, + }, }; +// Block +// TODO: split these tests into the disk_format sub-modules + #[test] fn roundtrip_block_height() { zebra_test::init(); @@ -29,6 +38,22 @@ fn roundtrip_block_height() { ); } +#[test] +fn roundtrip_block_hash() { + zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + +#[test] +fn roundtrip_block_header() { + zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + +// Transaction + #[test] fn roundtrip_transaction_location() { zebra_test::init(); @@ -41,27 +66,85 @@ fn roundtrip_transaction_location() { ); } +#[test] +fn roundtrip_transaction_hash() { + zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + +#[test] +fn roundtrip_transaction() { + zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + +// Transparent + +// TODO: turn this into a generic function like assert_value_properties() +#[test] +fn serialized_transparent_address_equal() { + zebra_test::init(); + + proptest!(|(val1 in any::(), val2 in any::())| { + if val1 == val2 { + prop_assert_eq!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were equal, but serialized bytes were not.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } else { + prop_assert_ne!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were not equal, but serialized bytes were equal.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } + } + ); +} + +#[test] +fn roundtrip_transparent_address() { + zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + #[test] fn roundtrip_output_location() { zebra_test::init(); + proptest!(|(val in any::())| assert_value_properties(val)); } #[test] -fn roundtrip_block_hash() { +fn roundtrip_address_location() { zebra_test::init(); - proptest!(|(val in any::())| assert_value_properties(val)); + + proptest!(|(val in any::())| assert_value_properties(val)); } #[test] -fn roundtrip_block_header() { +fn roundtrip_address_balance_location() { zebra_test::init(); - proptest!(|(val in any::())| assert_value_properties(val)); + proptest!(|(val in any::())| assert_value_properties(val)); } #[test] -fn roundtrip_transparent_output() { +fn roundtrip_unspent_transparent_output() { zebra_test::init(); proptest!( @@ -72,6 +155,228 @@ fn roundtrip_transparent_output() { ); } +#[test] +fn roundtrip_transparent_output() { + zebra_test::init(); + + proptest!(|(val in any::())| assert_value_properties(val)); +} + +#[test] +fn roundtrip_amount() { + zebra_test::init(); + + proptest!(|(val in any::>())| assert_value_properties(val)); +} + +// Sprout + +#[test] +fn serialized_sprout_nullifier_equal() { + zebra_test::init(); + + proptest!(|(val1 in any::(), val2 in any::())| { + if val1 == val2 { + prop_assert_eq!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were equal, but serialized bytes were not.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } else { + prop_assert_ne!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were not equal, but serialized bytes were equal.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } + } + ); +} + +#[test] +fn serialized_sprout_tree_root_equal() { + zebra_test::init(); + + proptest!(|(val1 in any::(), val2 in any::())| { + if val1 == val2 { + prop_assert_eq!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were equal, but serialized bytes were not.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } else { + prop_assert_ne!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were not equal, but serialized bytes were equal.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } + } + ); +} + +// TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary + +// Sapling + +#[test] +fn serialized_sapling_nullifier_equal() { + zebra_test::init(); + + proptest!(|(val1 in any::(), val2 in any::())| { + if val1 == val2 { + prop_assert_eq!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were equal, but serialized bytes were not.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } else { + prop_assert_ne!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were not equal, but serialized bytes were equal.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } + } + ); +} + +#[test] +fn serialized_sapling_tree_root_equal() { + zebra_test::init(); + + proptest!(|(val1 in any::(), val2 in any::())| { + if val1 == val2 { + prop_assert_eq!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were equal, but serialized bytes were not.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } else { + prop_assert_ne!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were not equal, but serialized bytes were equal.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } + } + ); +} + +// TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary + +// Orchard + +#[test] +fn serialized_orchard_nullifier_equal() { + zebra_test::init(); + + proptest!(|(val1 in any::(), val2 in any::())| { + if val1 == val2 { + prop_assert_eq!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were equal, but serialized bytes were not.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } else { + prop_assert_ne!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were not equal, but serialized bytes were equal.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } + } + ); +} + +#[test] +fn serialized_orchard_tree_root_equal() { + zebra_test::init(); + + proptest!(|(val1 in any::(), val2 in any::())| { + if val1 == val2 { + prop_assert_eq!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were equal, but serialized bytes were not.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } else { + prop_assert_ne!( + val1.as_bytes(), + val2.as_bytes(), + "struct values were not equal, but serialized bytes were equal.\n\ + Values:\n\ + {:?}\n\ + {:?}", + val1, + val2, + ); + } + } + ); +} + +// TODO: test note commitment tree round-trip, after implementing proptest::Arbitrary + +// Chain + +// TODO: test NonEmptyHistoryTree round-trip, after implementing proptest::Arbitrary + #[test] fn roundtrip_value_balance() { zebra_test::init(); diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap new file mode 100644 index 00000000..1aa792a1 --- /dev/null +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_1.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "017d46a730d31f97b1930d3368a967c309bd4d136a", + v: "d4300000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", + ), +] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap new file mode 100644 index 00000000..678849fb --- /dev/null +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@mainnet_2.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "017d46a730d31f97b1930d3368a967c309bd4d136a", + v: "7c920000000000000946edb9c083c9942d92305444527765fad789c438c717783276a9f7fbf61b8501000000", + ), +] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap new file mode 100644 index 00000000..0712b625 --- /dev/null +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_1.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642", + v: "d430000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", + ), +] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap new file mode 100644 index 00000000..124789a5 --- /dev/null +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/balance_by_transparent_addr_raw_data@testnet_2.snap @@ -0,0 +1,10 @@ +--- +source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +expression: cf_data +--- +[ + KV( + k: "03ef775f1f997f122a062fff1a2d7443abd1f9c642", + v: "7c92000000000000755f7c7d27a811596e9fae6dd30ca45be86e901d499909de35b6ff1f699f7ef301000000", + ), +] diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap index ab0fc72c..cde7c9aa 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/column_family_names.snap @@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs expression: cf_names --- [ + "balance_by_transparent_addr", "block_by_height", "default", "hash_by_height", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap index 075dd1b8..1b887c09 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@mainnet_0.snap @@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs expression: empty_column_families --- [ + "balance_by_transparent_addr: no entries", "history_tree: no entries", "orchard_anchors: no entries", "orchard_nullifiers: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap index 68d31a6b..f83855a2 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@no_blocks.snap @@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs expression: empty_column_families --- [ + "balance_by_transparent_addr: no entries", "block_by_height: no entries", "hash_by_height: no entries", "hash_by_tx_loc: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap index 075dd1b8..1b887c09 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshots/empty_column_families@testnet_0.snap @@ -3,6 +3,7 @@ source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs expression: empty_column_families --- [ + "balance_by_transparent_addr: no entries", "history_tree: no entries", "orchard_anchors: no entries", "orchard_nullifiers: no entries", diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs index cb0d5f4d..bcc44e16 100644 --- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs +++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs @@ -10,9 +10,12 @@ use std::fmt::Debug; use serde::{Deserialize, Serialize}; use zebra_chain::{ + amount::{Amount, NonNegative}, block::Height, + parameters::Network::*, serialization::{ZcashDeserializeInto, ZcashSerialize}, - transaction, transparent, + transaction, + transparent::{self, Address::*}, }; use crate::service::finalized_state::disk_format::{block::HEIGHT_DISK_BYTES, FromDisk, IntoDisk}; @@ -20,14 +23,17 @@ use crate::service::finalized_state::disk_format::{block::HEIGHT_DISK_BYTES, Fro #[cfg(any(test, feature = "proptest-impl"))] use proptest_derive::Arbitrary; +/// Transparent balances are stored as an 8 byte integer on disk. +pub const BALANCE_DISK_BYTES: usize = 8; + /// Output transaction locations are stored as a 32 byte transaction hash on disk. /// -/// TODO: change to TransactionLocation to reduce database size and increases lookup performance (#3151) +/// TODO: change to TransactionLocation to reduce database size and increases lookup performance (#3953) pub const OUTPUT_TX_HASH_DISK_BYTES: usize = 32; /// [`OutputIndex`]es are stored as 4 bytes on disk. /// -/// TODO: change to 3 bytes to reduce database size and increases lookup performance (#3151) +/// TODO: change to 3 bytes to reduce database size and increases lookup performance (#3953) pub const OUTPUT_INDEX_DISK_BYTES: usize = 4; // Transparent types @@ -76,6 +82,7 @@ impl OutputIndex { #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct OutputLocation { /// The transaction hash. + #[serde(with = "hex")] pub hash: transaction::Hash, /// The index of the transparent output in its transaction. @@ -102,10 +109,130 @@ impl OutputLocation { } } +/// The location of the first [`transparent::Output`] sent to an address. +/// +/// The address location stays the same, even if the corresponding output +/// has been spent. +/// +/// The first output location is used to represent the address in the database, +/// because output locations are significantly smaller than addresses. +/// +/// TODO: make this a different type to OutputLocation? +/// derive IntoDisk and FromDisk? +pub type AddressLocation = OutputLocation; + +/// Data which Zebra indexes for each [`transparent::Address`]. +/// +/// Currently, Zebra tracks this data 1:1 for each address: +/// - the balance [`Amount`] for a transparent address, and +/// - the [`OutputLocation`] for the first [`transparent::Output`] sent to that address +/// (regardless of whether that output is spent or unspent). +/// +/// All other address data is tracked multiple times for each address +/// (UTXOs and transactions). +#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] +pub struct AddressBalanceLocation { + /// The total balance of all UTXOs sent to an address. + balance: Amount, + + /// The location of the first [`transparent::Output`] sent to an address. + location: AddressLocation, +} + +impl AddressBalanceLocation { + /// Creates a new [`AddressBalanceLocation`] from the location of + /// the first [`transparent::Output`] sent to an address. + /// + /// The returned value has a zero initial balance. + pub fn new(first_output: OutputLocation) -> AddressBalanceLocation { + AddressBalanceLocation { + balance: Amount::zero(), + location: first_output, + } + } + + /// Returns the current balance for the address. + pub fn balance(&self) -> Amount { + self.balance + } + + /// Returns a mutable reference to the current balance for the address. + pub fn balance_mut(&mut self) -> &mut Amount { + &mut self.balance + } + + /// Returns the location of the first [`transparent::Output`] sent to an address. + pub fn location(&self) -> AddressLocation { + self.location + } +} + // Transparent trait impls -// TODO: serialize the index into a smaller number of bytes (#3152) -// serialize the index in big-endian order (#3150) +/// Returns a byte representing the [`transparent::Address`] variant. +fn address_variant(address: &transparent::Address) -> u8 { + // Return smaller values for more common variants. + // + // (This probably doesn't matter, but it might help slightly with data compression.) + match (address.network(), address) { + (Mainnet, PayToPublicKeyHash { .. }) => 0, + (Mainnet, PayToScriptHash { .. }) => 1, + (Testnet, PayToPublicKeyHash { .. }) => 2, + (Testnet, PayToScriptHash { .. }) => 3, + } +} + +impl IntoDisk for transparent::Address { + type Bytes = [u8; 21]; + + fn as_bytes(&self) -> Self::Bytes { + let variant_bytes = vec![address_variant(self)]; + let hash_bytes = self.hash_bytes().to_vec(); + + [variant_bytes, hash_bytes].concat().try_into().unwrap() + } +} + +#[cfg(any(test, feature = "proptest-impl"))] +impl FromDisk for transparent::Address { + fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { + let (address_variant, hash_bytes) = disk_bytes.as_ref().split_at(1); + + let address_variant = address_variant[0]; + let hash_bytes = hash_bytes.try_into().unwrap(); + + let network = if address_variant < 2 { + Mainnet + } else { + Testnet + }; + + if address_variant % 2 == 0 { + transparent::Address::from_pub_key_hash(network, hash_bytes) + } else { + transparent::Address::from_script_hash(network, hash_bytes) + } + } +} + +impl IntoDisk for Amount { + type Bytes = [u8; BALANCE_DISK_BYTES]; + + fn as_bytes(&self) -> Self::Bytes { + self.to_bytes() + } +} + +impl FromDisk for Amount { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + let array = bytes.as_ref().try_into().unwrap(); + Amount::from_bytes(array).unwrap() + } +} + +// TODO: serialize the index into a smaller number of bytes (#3953) +// serialize the index in big-endian order (#3953) impl IntoDisk for OutputIndex { type Bytes = [u8; OUTPUT_INDEX_DISK_BYTES]; @@ -142,7 +269,46 @@ impl FromDisk for OutputLocation { } } -// TODO: just serialize the Output, and derive the Utxo data from OutputLocation (#3151) +impl IntoDisk for AddressBalanceLocation { + type Bytes = [u8; BALANCE_DISK_BYTES + OUTPUT_TX_HASH_DISK_BYTES + OUTPUT_INDEX_DISK_BYTES]; + + fn as_bytes(&self) -> Self::Bytes { + let balance_bytes = self.balance().as_bytes().to_vec(); + let location_bytes = self.location().as_bytes().to_vec(); + + [balance_bytes, location_bytes].concat().try_into().unwrap() + } +} + +impl FromDisk for AddressBalanceLocation { + fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { + let (balance_bytes, location_bytes) = disk_bytes.as_ref().split_at(BALANCE_DISK_BYTES); + + let balance = Amount::from_bytes(balance_bytes.try_into().unwrap()).unwrap(); + let location = AddressLocation::from_bytes(location_bytes); + + let mut balance_location = AddressBalanceLocation::new(location); + *balance_location.balance_mut() = balance; + + balance_location + } +} + +impl IntoDisk for transparent::Output { + type Bytes = Vec; + + fn as_bytes(&self) -> Self::Bytes { + self.zcash_serialize_to_vec().unwrap() + } +} + +impl FromDisk for transparent::Output { + fn from_bytes(bytes: impl AsRef<[u8]>) -> Self { + bytes.as_ref().zcash_deserialize_into().unwrap() + } +} + +// TODO: delete UTXO serialization (#3953) impl IntoDisk for transparent::Utxo { type Bytes = Vec; diff --git a/zebra-state/src/service/finalized_state/zebra_db/arbitrary.rs b/zebra-state/src/service/finalized_state/zebra_db/arbitrary.rs index a3596d4e..1f2dc572 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/arbitrary.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/arbitrary.rs @@ -4,7 +4,9 @@ use std::ops::Deref; -use zebra_chain::{amount::NonNegative, block::Block, sprout, value_balance::ValueBalance}; +use zebra_chain::{ + amount::NonNegative, block::Block, parameters::Network::*, sprout, value_balance::ValueBalance, +}; use crate::service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, WriteDisk}, @@ -31,7 +33,7 @@ impl ZebraDb { /// Allow to set up a fake value pool in the database for testing purposes. pub fn set_finalized_value_pool(&self, fake_value_pool: ValueBalance) { - let mut batch = DiskWriteBatch::new(); + let mut batch = DiskWriteBatch::new(Mainnet); let value_pool_cf = self.db().cf_handle("tip_chain_value_pool").unwrap(); batch.zs_insert(&value_pool_cf, (), fake_value_pool); @@ -41,7 +43,7 @@ impl ZebraDb { /// Artificially prime the note commitment tree anchor sets with anchors /// referenced in a block, for testing purposes _only_. pub fn populate_with_anchors(&self, block: &Block) { - let mut batch = DiskWriteBatch::new(); + let mut batch = DiskWriteBatch::new(Mainnet); let sprout_anchors = self.db().cf_handle("sprout_anchors").unwrap(); let sapling_anchors = self.db().cf_handle("sapling_anchors").unwrap(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 1292cb21..75b0189c 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -11,6 +11,8 @@ use std::{collections::HashMap, sync::Arc}; +use itertools::Itertools; + use zebra_chain::{ amount::NonNegative, block::{self, Block}, @@ -25,7 +27,7 @@ use zebra_chain::{ use crate::{ service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, - disk_format::{FromDisk, TransactionLocation}, + disk_format::{transparent::AddressBalanceLocation, FromDisk, TransactionLocation}, zebra_db::{metrics::block_precommit_metrics, shielded::NoteCommitmentTrees, ZebraDb}, FinalizedBlock, }, @@ -188,23 +190,40 @@ impl ZebraDb { let finalized_hash = finalized.hash; // Get a list of the spent UTXOs, before we delete any from the database - let all_utxos_spent_by_block = finalized + let all_utxos_spent_by_block: HashMap = finalized .block .transactions .iter() .flat_map(|tx| tx.inputs().iter()) .flat_map(|input| input.outpoint()) - .flat_map(|outpoint| self.utxo(&outpoint).map(|utxo| (outpoint, utxo))) + .map(|outpoint| { + ( + outpoint, + // Some utxos are spent in the same block, so they will be in `new_outputs` + self.utxo(&outpoint) + .or_else(|| finalized.new_outputs.get(&outpoint).cloned()) + .expect("already checked UTXO was in state or block"), + ) + }) .collect(); - let mut batch = DiskWriteBatch::new(); + // Get the current address balances, before the transactions in this block + let address_balances = all_utxos_spent_by_block + .values() + .chain(finalized.new_outputs.values()) + .filter_map(|utxo| utxo.output.address(network)) + .unique() + .filter_map(|address| Some((address, self.address_balance_location(&address)?))) + .collect(); + + let mut batch = DiskWriteBatch::new(network); // In case of errors, propagate and do not write the batch. batch.prepare_block_batch( &self.db, finalized, - network, all_utxos_spent_by_block, + address_balances, self.note_commitment_trees(), history_tree, self.finalized_value_pool(), @@ -235,9 +254,8 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: FinalizedBlock, - network: Network, all_utxos_spent_by_block: HashMap, - // TODO: make an argument struct for all the current note commitment trees & history + address_balances: HashMap, mut note_commitment_trees: NoteCommitmentTrees, history_tree: HistoryTree, value_pool: ValueBalance, @@ -267,16 +285,16 @@ impl DiskWriteBatch { } // Commit transaction indexes - self.prepare_transaction_index_batch(db, &finalized, &mut note_commitment_trees)?; - - self.prepare_note_commitment_batch( + self.prepare_transaction_index_batch( db, &finalized, - network, - note_commitment_trees, - history_tree, + &all_utxos_spent_by_block, + address_balances, + &mut note_commitment_trees, )?; + self.prepare_note_commitment_batch(db, &finalized, note_commitment_trees, history_tree)?; + // Commit UTXOs and value pools self.prepare_chain_value_pools_batch(db, &finalized, all_utxos_spent_by_block, value_pool)?; @@ -378,6 +396,8 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, + all_utxos_spent_by_block: &HashMap, + address_balances: HashMap, note_commitment_trees: &mut NoteCommitmentTrees, ) -> Result<(), BoxError> { let FinalizedBlock { block, .. } = finalized; @@ -389,6 +409,11 @@ impl DiskWriteBatch { DiskWriteBatch::update_note_commitment_trees(transaction, note_commitment_trees)?; } - self.prepare_transparent_outputs_batch(db, finalized) + self.prepare_transparent_outputs_batch( + db, + finalized, + all_utxos_spent_by_block, + address_balances, + ) } } diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs index aa400cc0..2840dead 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs @@ -46,7 +46,9 @@ use zebra_chain::{ use crate::{ service::finalized_state::{ - disk_format::{block::TransactionIndex, transparent::OutputLocation, TransactionLocation}, + disk_format::{ + block::TransactionIndex, transparent::OutputLocation, FromDisk, TransactionLocation, + }, FinalizedState, }, Config, @@ -194,6 +196,7 @@ fn test_block_and_transaction_data_with_network(network: Network) { settings.set_snapshot_suffix(format!("{}_{}", net_suffix, height)); settings.bind(|| snapshot_block_and_transaction_data(&state)); + settings.bind(|| snapshot_transparent_address_data(&state)); } } @@ -216,17 +219,23 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { assert_eq!(sapling_tree, sapling::tree::NoteCommitmentTree::default()); assert_eq!(orchard_tree, orchard::tree::NoteCommitmentTree::default()); + // Blocks let mut stored_block_hashes = Vec::new(); let mut stored_blocks = Vec::new(); - let mut stored_sapling_trees = Vec::new(); - let mut stored_orchard_trees = Vec::new(); - + // Transactions let mut stored_transaction_hashes = Vec::new(); let mut stored_transactions = Vec::new(); + // Transparent + let mut stored_utxos = Vec::new(); + // Shielded + + let mut stored_sapling_trees = Vec::new(); + let mut stored_orchard_trees = Vec::new(); + let sapling_tree_at_tip = state.sapling_note_commitment_tree(); let orchard_tree_at_tip = state.orchard_note_commitment_tree(); @@ -412,6 +421,47 @@ fn snapshot_block_and_transaction_data(state: &FinalizedState) { } } +/// Snapshot transparent address data, using `cargo insta` and RON serialization. +fn snapshot_transparent_address_data(state: &FinalizedState) { + let balance_by_transparent_addr = state.cf_handle("balance_by_transparent_addr").unwrap(); + + let mut stored_address_balances = Vec::new(); + + // TODO: UTXOs for each address (#3953) + // transactions for each address (#3951) + + // Correctness: Multi-key iteration causes hangs in concurrent code, but seems ok in tests. + let addresses = + state.full_iterator_cf(&balance_by_transparent_addr, rocksdb::IteratorMode::Start); + + // The default raw data serialization is very verbose, so we hex-encode the bytes. + let addresses: Vec = addresses + .map(|(key, _value)| transparent::Address::from_bytes(key)) + .collect(); + + for address in addresses { + let stored_address_balance = state + .address_balance_location(&address) + .expect("address indexes are consistent"); + + stored_address_balances.push((address.to_string(), stored_address_balance)); + } + + // TODO: check that the UTXO and transaction lists are in chain order. + /* + assert!( + is_sorted(&stored_address_utxos), + "unsorted: {:?}", + stored_address_utxos, + ); + */ + + // We want to snapshot the order in the database, + // because sometimes it is significant for performance or correctness. + // So we don't sort the vectors before snapshotting. + insta::assert_ron_snapshot!("address_balances", stored_address_balances); +} + /// Return true if `list` is sorted in ascending order. /// /// TODO: replace with Vec::is_sorted when it stabilises diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_0.snap new file mode 100644 index 00000000..7dbefb77 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_0.snap @@ -0,0 +1,5 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_address_balances +--- +[] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap new file mode 100644 index 00000000..4fa05a94 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_1.snap @@ -0,0 +1,13 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_address_balances +--- +[ + ("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation( + balance: Amount(12500), + location: OutputLocation( + hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", + index: OutputIndex(1), + ), + )), +] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap new file mode 100644 index 00000000..fbd8f2ba --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@mainnet_2.snap @@ -0,0 +1,13 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_address_balances +--- +[ + ("t3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd", AddressBalanceLocation( + balance: Amount(37500), + location: OutputLocation( + hash: "851bf6fbf7a976327817c738c489d7fa657752445430922d94c983c0b9ed4609", + index: OutputIndex(1), + ), + )), +] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_0.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_0.snap new file mode 100644 index 00000000..7dbefb77 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_0.snap @@ -0,0 +1,5 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_address_balances +--- +[] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap new file mode 100644 index 00000000..843eb4fd --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_1.snap @@ -0,0 +1,13 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_address_balances +--- +[ + ("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation( + balance: Amount(12500), + location: OutputLocation( + hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", + index: OutputIndex(1), + ), + )), +] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap new file mode 100644 index 00000000..3f29fd70 --- /dev/null +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshots/address_balances@testnet_2.snap @@ -0,0 +1,13 @@ +--- +source: zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +expression: stored_address_balances +--- +[ + ("t2UNzUUx8mWBCRYPRezvA363EYXyEpHokyi", AddressBalanceLocation( + balance: Amount(37500), + location: OutputLocation( + hash: "f37e9f691fffb635de0999491d906ee85ba40cd36dae9f6e5911a8277d7c5f75", + index: OutputIndex(1), + ), + )), +] diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs index 987decc2..71890bc4 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/vectors.rs @@ -115,7 +115,7 @@ fn test_block_db_round_trip_with( }; // Skip validation by writing the block directly to the database - let mut batch = DiskWriteBatch::new(); + let mut batch = DiskWriteBatch::new(Mainnet); batch .prepare_block_header_transactions_batch(&state.db, &finalized) .expect("block is valid for batch"); diff --git a/zebra-state/src/service/finalized_state/zebra_db/chain.rs b/zebra-state/src/service/finalized_state/zebra_db/chain.rs index 6126077d..231859ed 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/chain.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/chain.rs @@ -16,9 +16,7 @@ use std::{borrow::Borrow, collections::HashMap}; use zebra_chain::{ amount::NonNegative, history_tree::{HistoryTree, NonEmptyHistoryTree}, - orchard, - parameters::Network, - sapling, transparent, + orchard, sapling, transparent, value_balance::ValueBalance, }; @@ -73,7 +71,6 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, - network: Network, sapling_root: sapling::tree::Root, orchard_root: orchard::tree::Root, mut history_tree: HistoryTree, @@ -82,7 +79,7 @@ impl DiskWriteBatch { let FinalizedBlock { block, height, .. } = finalized; - history_tree.push(network, block.clone(), sapling_root, orchard_root)?; + history_tree.push(self.network(), block.clone(), sapling_root, orchard_root)?; // Update the tree in state let current_tip_height = *height - 1; @@ -115,17 +112,12 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, - mut all_utxos_spent_by_block: HashMap, + all_utxos_spent_by_block: HashMap, value_pool: ValueBalance, ) -> Result<(), BoxError> { let tip_chain_value_pool = db.cf_handle("tip_chain_value_pool").unwrap(); - let FinalizedBlock { - block, new_outputs, .. - } = finalized; - - // Some utxos are spent in the same block, so they will be in `new_outputs`. - all_utxos_spent_by_block.extend(new_outputs.clone()); + let FinalizedBlock { block, .. } = finalized; let new_pool = value_pool.add_block(block.borrow(), &all_utxos_spent_by_block)?; self.zs_insert(&tip_chain_value_pool, (), new_pool); diff --git a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs index 0bfe4e4c..30b77e96 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/shielded.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/shielded.rs @@ -13,8 +13,7 @@ //! be incremented each time the database format (column, serialization, etc) changes. use zebra_chain::{ - block::Height, history_tree::HistoryTree, orchard, parameters::Network, sapling, sprout, - transaction::Transaction, + block::Height, history_tree::HistoryTree, orchard, sapling, sprout, transaction::Transaction, }; use crate::{ @@ -241,8 +240,6 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, - network: Network, - // TODO: make an argument struct for all the note commitment trees & history note_commitment_trees: NoteCommitmentTrees, history_tree: HistoryTree, ) -> Result<(), BoxError> { @@ -294,14 +291,7 @@ impl DiskWriteBatch { note_commitment_trees.orchard, ); - self.prepare_history_batch( - db, - finalized, - network, - sapling_root, - orchard_root, - history_tree, - ) + self.prepare_history_batch(db, finalized, sapling_root, orchard_root, history_tree) } /// Prepare a database batch containing the initial note commitment trees, diff --git a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs index 8065e796..8bbe73b7 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/transparent.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/transparent.rs @@ -11,14 +11,17 @@ //! The [`crate::constants::DATABASE_FORMAT_VERSION`] constant must //! be incremented each time the database format (column, serialization, etc) changes. -use std::borrow::Borrow; +use std::{borrow::Borrow, collections::HashMap}; -use zebra_chain::transparent; +use zebra_chain::{ + amount::{Amount, NonNegative}, + transparent, +}; use crate::{ service::finalized_state::{ disk_db::{DiskDb, DiskWriteBatch, ReadDisk, WriteDisk}, - disk_format::transparent::OutputLocation, + disk_format::transparent::{AddressBalanceLocation, AddressLocation, OutputLocation}, zebra_db::ZebraDb, FinalizedBlock, }, @@ -28,8 +31,37 @@ use crate::{ impl ZebraDb { // Read transparent methods - /// Returns the `transparent::Output` pointed to by the given - /// `transparent::OutPoint` if it is present. + /// Returns the [`AddressBalanceLocation`] for a [`transparent::Address`], + /// if it is in the finalized state. + pub fn address_balance_location( + &self, + address: &transparent::Address, + ) -> Option { + let balance_by_transparent_addr = self.db.cf_handle("balance_by_transparent_addr").unwrap(); + + self.db.zs_get(&balance_by_transparent_addr, address) + } + + /// Returns the balance for a [`transparent::Address`], + /// if it is in the finalized state. + #[allow(dead_code)] + pub fn address_balance(&self, address: &transparent::Address) -> Option> { + self.address_balance_location(address) + .map(|abl| abl.balance()) + } + + /// Returns the first output that sent funds to a [`transparent::Address`], + /// if it is in the finalized state. + /// + /// This location is used as an efficient index key for addresses. + #[allow(dead_code)] + pub fn address_location(&self, address: &transparent::Address) -> Option { + self.address_balance_location(address) + .map(|abl| abl.location()) + } + + /// Returns the transparent output for a [`transparent::OutPoint`], + /// if it is still unspent in the finalized state. pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option { let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap(); @@ -40,7 +72,11 @@ impl ZebraDb { } impl DiskWriteBatch { - /// Prepare a database batch containing `finalized.block`'s UTXO changes, + /// Prepare a database batch containing `finalized.block`'s: + /// - transparent address balance changes, + /// TODO: + /// - transparent address index changes (add in #3951, #3953), and + /// - UTXO changes (modify in #3952) /// and return it (without actually writing anything). /// /// # Errors @@ -50,8 +86,11 @@ impl DiskWriteBatch { &mut self, db: &DiskDb, finalized: &FinalizedBlock, + all_utxos_spent_by_block: &HashMap, + mut address_balances: HashMap, ) -> Result<(), BoxError> { let utxo_by_outpoint = db.cf_handle("utxo_by_outpoint").unwrap(); + let balance_by_transparent_addr = db.cf_handle("balance_by_transparent_addr").unwrap(); let FinalizedBlock { block, new_outputs, .. @@ -59,25 +98,61 @@ impl DiskWriteBatch { // Index all new transparent outputs, before deleting any we've spent for (outpoint, utxo) in new_outputs.borrow().iter() { + let receiving_address = utxo.output.address(self.network()); let output_location = OutputLocation::from_outpoint(outpoint); + // Update the address balance by adding this UTXO's value + if let Some(receiving_address) = receiving_address { + let address_balance = address_balances + .entry(receiving_address) + .or_insert_with(|| AddressBalanceLocation::new(output_location)) + .balance_mut(); + + let new_address_balance = (*address_balance + utxo.output.value()) + .expect("balance overflow already checked"); + + *address_balance = new_address_balance; + } + self.zs_insert(&utxo_by_outpoint, output_location, utxo); } // Mark all transparent inputs as spent. // - // Coinbase inputs represent new coins, - // so there are no UTXOs to mark as spent. - for output_location in block + // Coinbase inputs represent new coins, so there are no UTXOs to mark as spent. + for outpoint in block .transactions .iter() .flat_map(|tx| tx.inputs()) .flat_map(|input| input.outpoint()) - .map(|outpoint| OutputLocation::from_outpoint(&outpoint)) { + let output_location = OutputLocation::from_outpoint(&outpoint); + + let spent_output = &all_utxos_spent_by_block.get(&outpoint).unwrap().output; + let sending_address = spent_output.address(self.network()); + + // Update the address balance by subtracting this UTXO's value + if let Some(sending_address) = sending_address { + let address_balance = address_balances + .entry(sending_address) + .or_insert_with(|| panic!("spent outputs must already have an address balance")) + .balance_mut(); + + let new_address_balance = (*address_balance - spent_output.value()) + .expect("balance underflow already checked"); + + *address_balance = new_address_balance; + } + self.zs_delete(&utxo_by_outpoint, output_location); } + // Write the new address balances to the database + for (address, address_balance) in address_balances.into_iter() { + // Some of these balances are new, and some are updates + self.zs_insert(&balance_by_transparent_addr, address, address_balance); + } + Ok(()) } }