add option to disable rocksdb checksums

reference runtime state for default option initialization

Signed-off-by: Jason Volk <jason@zemos.net>

conduwuit-example.toml

@@ -897,6 +897,13 @@
 #
 #rocksdb_paranoid_file_checks = false
 
+# Enables or disables checksum verification in rocksdb at runtime.
+# Checksums are usually hardware accelerated with low overhead; they are
+# enabled in rocksdb by default. Older or slower platforms may see gains
+# from disabling.
+#
+#rocksdb_checksums = true
+
 # Database repair mode (for RocksDB SST corruption).
 #
 # Use this option when the server reports corruption while running or

src/core/config/mod.rs

@@ -1049,6 +1049,15 @@ pub struct Config {
 	#[serde(default)]
 	pub rocksdb_paranoid_file_checks: bool,
 
+	/// Enables or disables checksum verification in rocksdb at runtime.
+	/// Checksums are usually hardware accelerated with low overhead; they are
+	/// enabled in rocksdb by default. Older or slower platforms may see gains
+	/// from disabling.
+	///
+	/// default: true
+	#[serde(default = "true_fn")]
+	pub rocksdb_checksums: bool,
+
 	/// Database repair mode (for RocksDB SST corruption).
 	///
 	/// Use this option when the server reports corruption while running or

src/database/engine.rs

@@ -32,6 +32,7 @@ use crate::{
 pub struct Engine {
 	pub(super) read_only: bool,
 	pub(super) secondary: bool,
+	pub(crate) checksums: bool,
 	corks: AtomicU32,
 	pub(crate) db: Db,
 	pub(crate) pool: Arc<Pool>,

src/database/engine/open.rs

@@ -58,6 +58,7 @@ pub(crate) async fn open(ctx: Arc<Context>, desc: &[Descriptor]) -> Result<Arc<S
 	Ok(Arc::new(Self {
 		read_only: config.rocksdb_read_only,
 		secondary: config.rocksdb_secondary,
+		checksums: config.rocksdb_checksums,
 		corks: AtomicU32::new(0),
 		pool: ctx.pool.clone(),
 		db,

src/database/map.rs

@@ -59,9 +59,9 @@ impl Map {
 			db: db.clone(),
 			cf: open::open(db, name),
 			watchers: Watchers::default(),
-			write_options: write_options_default(),
-			read_options: read_options_default(),
-			cache_read_options: cache_read_options_default(),
+			write_options: write_options_default(db),
+			read_options: read_options_default(db),
+			cache_read_options: cache_read_options_default(db),
 		}))
 	}
 

src/database/map/keys.rs

@@ -22,7 +22,7 @@ where
 pub fn raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self) {
 		let state = state.init_fwd(None);

src/database/map/keys_from.rs

@@ -53,7 +53,7 @@ where
 {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self, from) {
 		return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed();

src/database/map/options.rs

@@ -1,35 +1,43 @@
+use std::sync::Arc;
+
 use rocksdb::{ReadOptions, ReadTier, WriteOptions};
 
-#[inline]
-pub(crate) fn iter_options_default() -> ReadOptions {
-	let mut options = read_options_default();
-	options.set_background_purge_on_iterator_cleanup(true);
-	//options.set_pin_data(true);
-	options
-}
+use crate::Engine;
 
 #[inline]
-pub(crate) fn cache_iter_options_default() -> ReadOptions {
-	let mut options = cache_read_options_default();
-	options.set_background_purge_on_iterator_cleanup(true);
-	//options.set_pin_data(true);
-	options
-}
-
-#[inline]
-pub(crate) fn cache_read_options_default() -> ReadOptions {
-	let mut options = read_options_default();
+pub(crate) fn cache_iter_options_default(db: &Arc<Engine>) -> ReadOptions {
+	let mut options = iter_options_default(db);
 	options.set_read_tier(ReadTier::BlockCache);
 	options.fill_cache(false);
 	options
 }
 
 #[inline]
-pub(crate) fn read_options_default() -> ReadOptions {
-	let mut options = ReadOptions::default();
-	options.set_total_order_seek(true);
+pub(crate) fn iter_options_default(db: &Arc<Engine>) -> ReadOptions {
+	let mut options = read_options_default(db);
+	options.set_background_purge_on_iterator_cleanup(true);
 	options
 }
 
 #[inline]
-pub(crate) fn write_options_default() -> WriteOptions { WriteOptions::default() }
+pub(crate) fn cache_read_options_default(db: &Arc<Engine>) -> ReadOptions {
+	let mut options = read_options_default(db);
+	options.set_read_tier(ReadTier::BlockCache);
+	options.fill_cache(false);
+	options
+}
+
+#[inline]
+pub(crate) fn read_options_default(db: &Arc<Engine>) -> ReadOptions {
+	let mut options = ReadOptions::default();
+	options.set_total_order_seek(true);
+
+	if !db.checksums {
+		options.set_verify_checksums(false);
+	}
+
+	options
+}
+
+#[inline]
+pub(crate) fn write_options_default(_db: &Arc<Engine>) -> WriteOptions { WriteOptions::default() }

src/database/map/rev_keys.rs

@@ -22,7 +22,7 @@ where
 pub fn rev_raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self) {
 		let state = state.init_rev(None);

src/database/map/rev_keys_from.rs

@@ -61,7 +61,7 @@ where
 {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self, from) {
 		return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed();

src/database/map/rev_stream.rs

@@ -31,7 +31,7 @@ where
 pub fn rev_raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self) {
 		let state = state.init_rev(None);
@@ -66,7 +66,7 @@ pub fn rev_raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>
     fields(%map),
 )]
 pub(super) fn is_cached(map: &Arc<super::Map>) -> bool {
-	let opts = super::cache_iter_options_default();
+	let opts = super::cache_iter_options_default(&map.db);
 	let state = stream::State::new(map, opts).init_rev(None);
 
 	!state.is_incomplete()

src/database/map/rev_stream_from.rs

@@ -80,7 +80,7 @@ where
 {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self, from) {
 		let state = state.init_rev(from.as_ref().into());
@@ -118,7 +118,7 @@ pub(super) fn is_cached<P>(map: &Arc<super::Map>, from: &P) -> bool
 where
 	P: AsRef<[u8]> + ?Sized,
 {
-	let cache_opts = super::cache_iter_options_default();
+	let cache_opts = super::cache_iter_options_default(&map.db);
 	let cache_status = stream::State::new(map, cache_opts)
 		.init_rev(from.as_ref().into())
 		.status();

src/database/map/stream.rs

@@ -30,7 +30,7 @@ where
 pub fn raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self) {
 		let state = state.init_fwd(None);
@@ -65,7 +65,7 @@ pub fn raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> +
     fields(%map),
 )]
 pub(super) fn is_cached(map: &Arc<super::Map>) -> bool {
-	let opts = super::cache_iter_options_default();
+	let opts = super::cache_iter_options_default(&map.db);
 	let state = stream::State::new(map, opts).init_fwd(None);
 
 	!state.is_incomplete()

src/database/map/stream_from.rs

@@ -77,7 +77,7 @@ where
 {
 	use crate::pool::Seek;
 
-	let opts = super::iter_options_default();
+	let opts = super::iter_options_default(&self.db);
 	let state = stream::State::new(self, opts);
 	if is_cached(self, from) {
 		let state = state.init_fwd(from.as_ref().into());
@@ -115,7 +115,7 @@ pub(super) fn is_cached<P>(map: &Arc<super::Map>, from: &P) -> bool
 where
 	P: AsRef<[u8]> + ?Sized,
 {
-	let opts = super::cache_iter_options_default();
+	let opts = super::cache_iter_options_default(&map.db);
 	let state = stream::State::new(map, opts).init_fwd(from.as_ref().into());
 
 	!state.is_incomplete()