diff --git a/Cargo.lock b/Cargo.lock index 332b59dc..38b33b32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -431,6 +431,25 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.60.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2 1.0.42", + "quote 1.0.20", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bip0039" version = "0.9.0" @@ -2506,11 +2525,11 @@ checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" [[package]] name = "librocksdb-sys" -version = "0.6.1+6.28.2" +version = "0.8.0+7.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc587013734dadb7cf23468e531aa120788b87243648be42e2d3a072186291" +checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d" dependencies = [ - "bindgen", + "bindgen 0.60.1", "bzip2-sys", "cc", "glob", @@ -3997,9 +4016,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "620f4129485ff1a7128d184bc687470c21c7951b64779ebc9cfdad3dcd920290" +checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc" dependencies = [ "libc", "librocksdb-sys", @@ -6291,7 +6310,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e4255f320dead417a91cbf00a178b0d702b813d5a8c95a5f2e4cc7ccced17f" dependencies = [ - "bindgen", + "bindgen 0.59.2", "blake2b_simd", "cc", "cxx", diff --git a/deny.toml b/deny.toml index 6259ed37..d80916b0 100644 --- a/deny.toml +++ b/deny.toml @@ -78,6 +78,10 @@ skip-tree = [ # wait for primitive-types to upgrade { name = "proc-macro-crate", version = "=0.1.5" }, + + # wait for zcash_script to upgrade bindgen + # https://github.com/ZcashFoundation/zcash_script/issues/40 + { name = "bindgen", version = "=0.59.2" }, ] # This section is considered when running `cargo deny check sources`. diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index 337add60..d414167e 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -21,7 +21,7 @@ metrics = "0.18.1" mset = "0.1.0" regex = "1.6.0" rlimit = "0.8.3" -rocksdb = { version = "0.18.0", default_features = false, features = ["lz4"] } +rocksdb = { version = "0.19.0", default_features = false, features = ["lz4"] } serde = { version = "1.0.144", features = ["serde_derive"] } tempfile = "3.3.0" thiserror = "1.0.33" diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index dd458615..888e9b99 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -714,6 +714,7 @@ impl Service for StateService { let timer = CodeTimer::start(); // TODO: move this work into the future, like Block and Transaction? + // move disk reads to a blocking thread (#2188) let rsp = Ok(Response::Depth(self.best_depth(hash))); // The work is all done, the future just returns the result. @@ -734,6 +735,7 @@ impl Service for StateService { let timer = CodeTimer::start(); // TODO: move this work into the future, like Block and Transaction? + // move disk reads to a blocking thread (#2188) let rsp = Ok(Response::Tip(self.best_tip())); // The work is all done, the future just returns the result. @@ -752,6 +754,7 @@ impl Service for StateService { let timer = CodeTimer::start(); // TODO: move this work into the future, like Block and Transaction? + // move disk reads to a blocking thread (#2188) let rsp = Ok(Response::BlockLocator( self.block_locator().unwrap_or_default(), )); @@ -836,6 +839,7 @@ impl Service for StateService { let fut = self.pending_utxos.queue(outpoint); + // TODO: move disk reads (in `any_utxo()`) to a blocking thread (#2188) if let Some(utxo) = self.any_utxo(&outpoint) { self.pending_utxos.respond(&outpoint, utxo); } diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index ef3434d6..db7c628e 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -210,10 +210,10 @@ impl ReadDisk for DiskDb { // Empty column families return invalid forward iterators. // // Checking iterator validity does not seem to cause database hangs. - !self - .db - .iterator_cf(cf, rocksdb::IteratorMode::Start) - .valid() + let iterator = self.db.iterator_cf(cf, rocksdb::IteratorMode::Start); + let raw_iterator: rocksdb::DBRawIteratorWithThreadMode = iterator.into(); + + !raw_iterator.valid() } #[allow(clippy::unwrap_in_result)] @@ -228,12 +228,10 @@ impl ReadDisk for DiskDb { // We use `get_pinned_cf` to avoid taking ownership of the serialized // value, because we're going to deserialize it anyways, which avoids an // extra copy - // - // TODO: move disk reads to a blocking thread (#2188) let value_bytes = self .db .get_pinned_cf(cf, key_bytes) - .expect("expected that disk errors would not occur"); + .expect("unexpected database failure"); value_bytes.map(V::from_bytes) } @@ -247,14 +245,13 @@ impl ReadDisk for DiskDb { // We use `get_pinned_cf` to avoid taking ownership of the serialized // value, because we don't use the value at all. This avoids an extra copy. - // - // TODO: move disk reads to a blocking thread (#2188) self.db .get_pinned_cf(cf, key_bytes) - .expect("expected that disk errors would not occur") + .expect("unexpected database failure") .is_some() } + #[allow(clippy::unwrap_in_result)] fn zs_first_key_value(&self, cf: &C) -> Option<(K, V)> where C: rocksdb::AsColumnFamilyRef, @@ -264,10 +261,14 @@ impl ReadDisk for DiskDb { // Reading individual values from iterators does not seem to cause database hangs. self.db .iterator_cf(cf, rocksdb::IteratorMode::Start) - .next() - .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + .next()? + .map(|(key_bytes, value_bytes)| { + Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + }) + .expect("unexpected database failure") } + #[allow(clippy::unwrap_in_result)] fn zs_last_key_value(&self, cf: &C) -> Option<(K, V)> where C: rocksdb::AsColumnFamilyRef, @@ -277,10 +278,14 @@ impl ReadDisk for DiskDb { // Reading individual values from iterators does not seem to cause database hangs. self.db .iterator_cf(cf, rocksdb::IteratorMode::End) - .next() - .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + .next()? + .map(|(key_bytes, value_bytes)| { + Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + }) + .expect("unexpected database failure") } + #[allow(clippy::unwrap_in_result)] fn zs_next_key_value_from(&self, cf: &C, lower_bound: &K) -> Option<(K, V)> where C: rocksdb::AsColumnFamilyRef, @@ -293,10 +298,14 @@ impl ReadDisk for DiskDb { // Reading individual values from iterators does not seem to cause database hangs. self.db .iterator_cf(cf, from) - .next() - .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + .next()? + .map(|(key_bytes, value_bytes)| { + Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + }) + .expect("unexpected database failure") } + #[allow(clippy::unwrap_in_result)] fn zs_prev_key_value_back_from(&self, cf: &C, upper_bound: &K) -> Option<(K, V)> where C: rocksdb::AsColumnFamilyRef, @@ -309,8 +318,11 @@ impl ReadDisk for DiskDb { // Reading individual values from iterators does not seem to cause database hangs. self.db .iterator_cf(cf, from) - .next() - .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + .next()? + .map(|(key_bytes, value_bytes)| { + Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes))) + }) + .expect("unexpected database failure") } } diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs index ac559e97..a1564096 100644 --- a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs @@ -36,7 +36,11 @@ use zebra_chain::{ }; use crate::{ - service::finalized_state::{disk_db::DiskDb, disk_format::tests::KV, FinalizedState}, + service::finalized_state::{ + disk_db::{DiskDb, DB}, + disk_format::tests::KV, + FinalizedState, + }, Config, }; @@ -129,6 +133,7 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str // The default raw data serialization is very verbose, so we hex-encode the bytes. let cf_data: Vec = cf_iter .by_ref() + .map(|result| result.expect("unexpected database error")) .map(|(key, value)| KV::new(key, value)) .collect(); @@ -144,8 +149,10 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str insta::assert_ron_snapshot!(format!("{}_raw_data", cf_name), cf_data); } + let raw_cf_iter: rocksdb::DBRawIteratorWithThreadMode = cf_iter.into(); + assert_eq!( - cf_iter.status(), + raw_cf_iter.status(), Ok(()), "unexpected column family iterator error", ); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs index 13644dff..15e190ac 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs @@ -462,6 +462,7 @@ fn snapshot_transparent_address_data(state: &FinalizedState, height: u32) { .count(); let addresses: Vec = addresses + .map(|result| result.expect("unexpected database error")) .map(|(key, _value)| transparent::Address::from_bytes(key)) .collect();