change(deps): bump rocksdb from 0.18.0 to 0.19.0 (#5071)

* cargo upgrade --workspace rocksdb

* Add a deny.toml exception for bindgen 0.59

* Move `valid()` and `status()` methods to raw iterators

* Update some outdated comments

* Panic on iterator failures

We might want to change this if there are any common failure modes.

* allow(clippy::unwrap_in_result) in some methods

Cargo.lock

@@ -431,6 +431,25 @@ dependencies = [
  "which",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.60.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "clang-sys",
+ "lazy_static",
+ "lazycell",
+ "peeking_take_while",
+ "proc-macro2 1.0.42",
+ "quote 1.0.20",
+ "regex",
+ "rustc-hash",
+ "shlex",
+]
+
 [[package]]
 name = "bip0039"
 version = "0.9.0"
@@ -2506,11 +2525,11 @@ checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db"
 
 [[package]]
 name = "librocksdb-sys"
-version = "0.6.1+6.28.2"
+version = "0.8.0+7.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bc587013734dadb7cf23468e531aa120788b87243648be42e2d3a072186291"
+checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d"
 dependencies = [
- "bindgen",
+ "bindgen 0.60.1",
  "bzip2-sys",
  "cc",
  "glob",
@@ -3997,9 +4016,9 @@ dependencies = [
 
 [[package]]
 name = "rocksdb"
-version = "0.18.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "620f4129485ff1a7128d184bc687470c21c7951b64779ebc9cfdad3dcd920290"
+checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc"
 dependencies = [
  "libc",
  "librocksdb-sys",
@@ -6291,7 +6310,7 @@ version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81e4255f320dead417a91cbf00a178b0d702b813d5a8c95a5f2e4cc7ccced17f"
 dependencies = [
- "bindgen",
+ "bindgen 0.59.2",
  "blake2b_simd",
  "cc",
  "cxx",

deny.toml

@@ -78,6 +78,10 @@ skip-tree = [
 
     # wait for primitive-types to upgrade
     { name = "proc-macro-crate", version = "=0.1.5" },
+
+    # wait for zcash_script to upgrade bindgen
+    # https://github.com/ZcashFoundation/zcash_script/issues/40
+    { name = "bindgen", version = "=0.59.2" },
 ]
 
 # This section is considered when running `cargo deny check sources`.

zebra-state/Cargo.toml

@@ -21,7 +21,7 @@ metrics = "0.18.1"
 mset = "0.1.0"
 regex = "1.6.0"
 rlimit = "0.8.3"
-rocksdb = { version = "0.18.0", default_features = false, features = ["lz4"] }
+rocksdb = { version = "0.19.0", default_features = false, features = ["lz4"] }
 serde = { version = "1.0.144", features = ["serde_derive"] }
 tempfile = "3.3.0"
 thiserror = "1.0.33"

zebra-state/src/service.rs

@@ -714,6 +714,7 @@ impl Service<Request> for StateService {
                 let timer = CodeTimer::start();
 
                 // TODO: move this work into the future, like Block and Transaction?
+                //       move disk reads to a blocking thread (#2188)
                 let rsp = Ok(Response::Depth(self.best_depth(hash)));
 
                 // The work is all done, the future just returns the result.
@@ -734,6 +735,7 @@ impl Service<Request> for StateService {
                 let timer = CodeTimer::start();
 
                 // TODO: move this work into the future, like Block and Transaction?
+                //       move disk reads to a blocking thread (#2188)
                 let rsp = Ok(Response::Tip(self.best_tip()));
 
                 // The work is all done, the future just returns the result.
@@ -752,6 +754,7 @@ impl Service<Request> for StateService {
                 let timer = CodeTimer::start();
 
                 // TODO: move this work into the future, like Block and Transaction?
+                //       move disk reads to a blocking thread (#2188)
                 let rsp = Ok(Response::BlockLocator(
                     self.block_locator().unwrap_or_default(),
                 ));
@@ -836,6 +839,7 @@ impl Service<Request> for StateService {
 
                 let fut = self.pending_utxos.queue(outpoint);
 
+                // TODO: move disk reads (in `any_utxo()`) to a blocking thread (#2188)
                 if let Some(utxo) = self.any_utxo(&outpoint) {
                     self.pending_utxos.respond(&outpoint, utxo);
                 }

zebra-state/src/service/finalized_state/disk_db.rs

@@ -210,10 +210,10 @@ impl ReadDisk for DiskDb {
         // Empty column families return invalid forward iterators.
         //
         // Checking iterator validity does not seem to cause database hangs.
-        !self
+        let iterator = self.db.iterator_cf(cf, rocksdb::IteratorMode::Start);
-            .db
+        let raw_iterator: rocksdb::DBRawIteratorWithThreadMode<DB> = iterator.into();
-            .iterator_cf(cf, rocksdb::IteratorMode::Start)
+
-            .valid()
+        !raw_iterator.valid()
     }
 
     #[allow(clippy::unwrap_in_result)]
@@ -228,12 +228,10 @@ impl ReadDisk for DiskDb {
         // We use `get_pinned_cf` to avoid taking ownership of the serialized
         // value, because we're going to deserialize it anyways, which avoids an
         // extra copy
-        //
-        // TODO: move disk reads to a blocking thread (#2188)
         let value_bytes = self
             .db
             .get_pinned_cf(cf, key_bytes)
-            .expect("expected that disk errors would not occur");
+            .expect("unexpected database failure");
 
         value_bytes.map(V::from_bytes)
     }
@@ -247,14 +245,13 @@ impl ReadDisk for DiskDb {
 
         // We use `get_pinned_cf` to avoid taking ownership of the serialized
         // value, because we don't use the value at all. This avoids an extra copy.
-        //
-        // TODO: move disk reads to a blocking thread (#2188)
         self.db
             .get_pinned_cf(cf, key_bytes)
-            .expect("expected that disk errors would not occur")
+            .expect("unexpected database failure")
             .is_some()
     }
 
+    #[allow(clippy::unwrap_in_result)]
     fn zs_first_key_value<C, K, V>(&self, cf: &C) -> Option<(K, V)>
     where
         C: rocksdb::AsColumnFamilyRef,
@@ -264,10 +261,14 @@ impl ReadDisk for DiskDb {
         // Reading individual values from iterators does not seem to cause database hangs.
         self.db
             .iterator_cf(cf, rocksdb::IteratorMode::Start)
-            .next()
+            .next()?
-            .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            .map(|(key_bytes, value_bytes)| {
+                Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            })
+            .expect("unexpected database failure")
     }
 
+    #[allow(clippy::unwrap_in_result)]
     fn zs_last_key_value<C, K, V>(&self, cf: &C) -> Option<(K, V)>
     where
         C: rocksdb::AsColumnFamilyRef,
@@ -277,10 +278,14 @@ impl ReadDisk for DiskDb {
         // Reading individual values from iterators does not seem to cause database hangs.
         self.db
             .iterator_cf(cf, rocksdb::IteratorMode::End)
-            .next()
+            .next()?
-            .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            .map(|(key_bytes, value_bytes)| {
+                Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            })
+            .expect("unexpected database failure")
     }
 
+    #[allow(clippy::unwrap_in_result)]
     fn zs_next_key_value_from<C, K, V>(&self, cf: &C, lower_bound: &K) -> Option<(K, V)>
     where
         C: rocksdb::AsColumnFamilyRef,
@@ -293,10 +298,14 @@ impl ReadDisk for DiskDb {
         // Reading individual values from iterators does not seem to cause database hangs.
         self.db
             .iterator_cf(cf, from)
-            .next()
+            .next()?
-            .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            .map(|(key_bytes, value_bytes)| {
+                Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            })
+            .expect("unexpected database failure")
     }
 
+    #[allow(clippy::unwrap_in_result)]
     fn zs_prev_key_value_back_from<C, K, V>(&self, cf: &C, upper_bound: &K) -> Option<(K, V)>
     where
         C: rocksdb::AsColumnFamilyRef,
@@ -309,8 +318,11 @@ impl ReadDisk for DiskDb {
         // Reading individual values from iterators does not seem to cause database hangs.
         self.db
             .iterator_cf(cf, from)
-            .next()
+            .next()?
-            .map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            .map(|(key_bytes, value_bytes)| {
+                Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
+            })
+            .expect("unexpected database failure")
     }
 }
 

zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs

@@ -36,7 +36,11 @@ use zebra_chain::{
 };
 
 use crate::{
-    service::finalized_state::{disk_db::DiskDb, disk_format::tests::KV, FinalizedState},
+    service::finalized_state::{
+        disk_db::{DiskDb, DB},
+        disk_format::tests::KV,
+        FinalizedState,
+    },
     Config,
 };
 
@@ -129,6 +133,7 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
         // The default raw data serialization is very verbose, so we hex-encode the bytes.
         let cf_data: Vec<KV> = cf_iter
             .by_ref()
+            .map(|result| result.expect("unexpected database error"))
             .map(|(key, value)| KV::new(key, value))
             .collect();
 
@@ -144,8 +149,10 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
             insta::assert_ron_snapshot!(format!("{}_raw_data", cf_name), cf_data);
         }
 
+        let raw_cf_iter: rocksdb::DBRawIteratorWithThreadMode<DB> = cf_iter.into();
+
         assert_eq!(
-            cf_iter.status(),
+            raw_cf_iter.status(),
             Ok(()),
             "unexpected column family iterator error",
         );

zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs

@@ -462,6 +462,7 @@ fn snapshot_transparent_address_data(state: &FinalizedState, height: u32) {
         .count();
 
     let addresses: Vec<transparent::Address> = addresses
+        .map(|result| result.expect("unexpected database error"))
         .map(|(key, _value)| transparent::Address::from_bytes(key))
         .collect();