add compression-shaping; tweak default compression levels

Signed-off-by: Jason Volk <jason@zemos.net>

conduwuit-example.toml

@@ -818,6 +818,9 @@
 # magic number and translated to the library's default compression level
 # as they all differ. See their `kDefaultCompressionLevel`.
 #
+# Note when using the default value we may override it with a setting
+# tailored specifically conduwuit.
+#
 #rocksdb_compression_level = 32767
 
 # Level of compression the specified compression algorithm for the
@@ -831,6 +834,9 @@
 # less likely for this data to be used. Research your chosen compression
 # algorithm.
 #
+# Note when using the default value we may override it with a setting
+# tailored specifically conduwuit.
+#
 #rocksdb_bottommost_compression_level = 32767
 
 # Whether to enable RocksDB's "bottommost_compression".
@@ -842,7 +848,7 @@
 #
 # See https://github.com/facebook/rocksdb/wiki/Compression for more details.
 #
-#rocksdb_bottommost_compression = false
+#rocksdb_bottommost_compression = true
 
 # Database recovery mode (for RocksDB WAL corruption).
 #

src/core/config/mod.rs

@@ -958,6 +958,9 @@ pub struct Config {
 	/// magic number and translated to the library's default compression level
 	/// as they all differ. See their `kDefaultCompressionLevel`.
 	///
+	/// Note when using the default value we may override it with a setting
+	/// tailored specifically conduwuit.
+	///
 	/// default: 32767
 	#[serde(default = "default_rocksdb_compression_level")]
 	pub rocksdb_compression_level: i32,
@@ -973,6 +976,9 @@ pub struct Config {
 	/// less likely for this data to be used. Research your chosen compression
 	/// algorithm.
 	///
+	/// Note when using the default value we may override it with a setting
+	/// tailored specifically conduwuit.
+	///
 	/// default: 32767
 	#[serde(default = "default_rocksdb_bottommost_compression_level")]
 	pub rocksdb_bottommost_compression_level: i32,
@@ -985,7 +991,7 @@ pub struct Config {
 	/// if you're trying to reduce storage usage from the database.
 	///
 	/// See https://github.com/facebook/rocksdb/wiki/Compression for more details.
-	#[serde(default)]
+	#[serde(default = "true_fn")]
 	pub rocksdb_bottommost_compression: bool,
 
 	/// Database recovery mode (for RocksDB WAL corruption).

src/database/engine/cf_opts.rs

@@ -8,6 +8,8 @@ use rocksdb::{
 use super::descriptor::{CacheDisp, Descriptor};
 use crate::{util::map_err, Context};
 
+pub(super) const SENTINEL_COMPRESSION_LEVEL: i32 = 32767;
+
 /// Adjust options for the specific column by name. Provide the result of
 /// db_options() as the argument to this function and use the return value in
 /// the arguments to open the specific column.
@@ -45,7 +47,15 @@ fn descriptor_cf_options(
 	opts.set_compaction_pri(desc.compaction_pri);
 	opts.set_universal_compaction_options(&uc_options(&desc));
 
+	let compression_shape: Vec<_> = desc
+		.compression_shape
+		.into_iter()
+		.map(|val| (val > 0).then_some(desc.compression))
+		.map(|val| val.unwrap_or(CompressionType::None))
+		.collect();
+
 	opts.set_compression_type(desc.compression);
+	opts.set_compression_per_level(compression_shape.as_slice());
 	opts.set_compression_options(-14, desc.compression_level, 0, 0); // -14 w_bits used by zlib.
 	if let Some(&bottommost_level) = desc.bottommost_level.as_ref() {
 		opts.set_bottommost_compression_type(desc.compression);
@@ -95,10 +105,24 @@ fn set_compression(desc: &mut Descriptor, config: &Config) {
 		| _ => CompressionType::Zstd,
 	};
 
-	desc.compression_level = config.rocksdb_compression_level;
-	desc.bottommost_level = config
-		.rocksdb_bottommost_compression
-		.then_some(config.rocksdb_bottommost_compression_level);
+	let can_override_level = config.rocksdb_compression_level == SENTINEL_COMPRESSION_LEVEL
+		&& desc.compression == CompressionType::Zstd;
+
+	if !can_override_level {
+		desc.compression_level = config.rocksdb_compression_level;
+	}
+
+	let can_override_bottom = config.rocksdb_bottommost_compression_level
+		== SENTINEL_COMPRESSION_LEVEL
+		&& desc.compression == CompressionType::Zstd;
+
+	if !can_override_bottom {
+		desc.bottommost_level = Some(config.rocksdb_bottommost_compression_level);
+	}
+
+	if !config.rocksdb_bottommost_compression {
+		desc.bottommost_level = None;
+	}
 }
 
 fn uc_options(desc: &Descriptor) -> UniversalCompactOptions {

src/database/engine/descriptor.rs

@@ -4,6 +4,8 @@ use rocksdb::{
 	DBCompressionType as CompressionType,
 };
 
+use super::cf_opts::SENTINEL_COMPRESSION_LEVEL;
+
 #[derive(Debug, Clone, Copy)]
 pub(crate) enum CacheDisp {
 	Unique,
@@ -32,6 +34,7 @@ pub(crate) struct Descriptor {
 	pub(crate) compaction: CompactionStyle,
 	pub(crate) compaction_pri: CompactionPri,
 	pub(crate) compression: CompressionType,
+	pub(crate) compression_shape: [i32; 7],
 	pub(crate) compression_level: i32,
 	pub(crate) bottommost_level: Option<i32>,
 	pub(crate) block_index_hashing: Option<bool>,
@@ -58,8 +61,9 @@ pub(crate) static BASE: Descriptor = Descriptor {
 	compaction: CompactionStyle::Level,
 	compaction_pri: CompactionPri::MinOverlappingRatio,
 	compression: CompressionType::Zstd,
-	compression_level: 32767,
-	bottommost_level: Some(32767),
+	compression_shape: [0, 0, 0, 1, 1, 1, 1],
+	compression_level: SENTINEL_COMPRESSION_LEVEL,
+	bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL),
 	block_index_hashing: None,
 	cache_shards: 64,
 };
@@ -68,6 +72,8 @@ pub(crate) static RANDOM: Descriptor = Descriptor {
 	compaction_pri: CompactionPri::OldestSmallestSeqFirst,
 	write_size: 1024 * 1024 * 32,
 	cache_shards: 128,
+	compression_level: -3,
+	bottommost_level: Some(4),
 	..BASE
 };
 
@@ -77,6 +83,9 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor {
 	level_size: 1024 * 1024 * 32,
 	file_size: 1024 * 1024 * 2,
 	cache_shards: 128,
+	compression_level: -1,
+	bottommost_level: Some(6),
+	compression_shape: [0, 0, 1, 1, 1, 1, 1],
 	..BASE
 };
 
@@ -88,6 +97,9 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor {
 	index_size: 512,
 	block_size: 512,
 	cache_shards: 64,
+	compression_level: -4,
+	bottommost_level: Some(1),
+	compression_shape: [0, 0, 0, 0, 0, 1, 1],
 	..RANDOM
 };
 
@@ -99,5 +111,8 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor {
 	block_size: 512,
 	cache_shards: 64,
 	block_index_hashing: Some(false),
+	compression_level: -2,
+	bottommost_level: Some(4),
+	compression_shape: [0, 0, 0, 0, 1, 1, 1],
 	..SEQUENTIAL
 };