// Sonic // // Fast, lightweight and schema-less search backend // Copyright: 2019, Valerian Saliou // Copyright: 2026, Rémi Bardon // License: Mozilla Public License v2.0 (MPL v2.0) //! Sonic library configuration. //! //! It does not include server nor channel configuration, which are specific //! to the `sonic-server` binary. use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use serde::Deserialize; use crate::util::serde::env_var; #[derive(Debug, Deserialize)] pub struct Config { pub normalization: NormalizationConfig, pub tokenization: TokenizationConfig, pub stopwords: StopwordsConfig, pub search: SearchConfig, pub store: StoreConfig, } impl Config { pub fn validate(&self) { // Check 'write_buffer' for KV if self.store.kv.database.write_buffer_size == Some(0) { panic!("write_buffer for kv must not be zero"); } // Check 'flush_after' for KV if self.store.kv.database.flush_after >= self.store.kv.pool.inactive_after { panic!("flush_after for kv must be strictly lower than inactive_after"); } // Check 'consolidate_after' for FST if self.store.fst.graph.consolidate_after >= self.store.fst.pool.inactive_after { panic!("consolidate_after for fst must be strictly lower than inactive_after"); } } } /// Configuration group for normalization options (Unicode normalization, /// stemming, lemmatization…). #[derive(Debug, Deserialize, Clone, Copy)] pub struct NormalizationConfig { #[serde(with = "crate::util::serde::none_string_as_none")] pub unicode_normalization: Option, pub diacritic_folding_enabled: bool, #[cfg(feature = "stemming")] pub stemming_enabled: bool, } #[derive(Deserialize, Debug, Clone, Copy)] #[serde(rename_all = "snake_case")] pub enum UnicodeNormalization { /// Unicode Normalization Form C. Nfc, /// Unicode Normalization Form KC. Nfkc, } /// Configuration group for tokenization options. #[derive(Debug, Deserialize, Clone, Copy)] pub struct TokenizationConfig { pub detect_special_patterns: bool, #[serde(alias = "split_special_patterns")] pub compat_split_special_patterns: bool, } #[derive(Debug, Deserialize, Clone, Default)] pub struct StopwordsConfig { #[serde(deserialize_with = "to_stopwords")] pub allow: HashSet, #[serde(deserialize_with = "to_stopwords")] pub deny: HashSet, } fn to_stopwords<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::de::Deserializer<'de>, { use unicode_normalization::UnicodeNormalization as _; let vec: Vec> = Deserialize::deserialize(deserializer)?; let stopwords_iter = vec.into_iter().map(|s| s.nfkd().to_string()); Ok(HashSet::from_iter(stopwords_iter)) } #[derive(Debug, Deserialize)] pub struct SearchConfig { pub query_limit_default: u16, pub query_limit_maximum: u16, pub query_alternates_try: usize, pub query_minimum_term_idf_default: f32, pub query_minimum_term_idf_minimum_object_count: u64, pub suggest_limit_default: u16, pub suggest_limit_maximum: u16, pub list_limit_default: u16, pub list_limit_maximum: u16, } #[derive(Debug, Deserialize)] pub struct StoreConfig { pub kv: Arc, pub fst: Arc, } #[derive(Debug, Deserialize)] pub struct KvStoreConfig { #[serde(deserialize_with = "env_var::path_buf")] pub path: PathBuf, pub retain_word_objects: usize, pub pool: KvStorePoolConfig, pub database: KvStoreDatabaseConfig, } #[derive(Debug, Deserialize)] pub struct KvStorePoolConfig { pub inactive_after: u64, } #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct KvStoreDatabaseConfig { pub flush_after: u64, pub write_ahead_log: bool, /// Whether or not to compress. /// /// Will get overriden if [`compression_type`](Self::compression_type) is /// also specified. #[serde(default)] pub compress: Option, #[serde(default)] pub parallelism: Option, #[serde(default)] #[serde(alias = "max_files")] pub max_open_files: Option, // TODO(major): Make this MB, as in Kvrocks. /// WARN: In KB! #[serde(default = "default_write_buffer_size")] #[serde(alias = "write_buffer")] pub write_buffer_size: Option, #[serde(default)] pub max_write_buffer_number: Option, #[serde(default)] pub min_write_buffer_number: Option, #[serde(default)] pub min_write_buffer_number_to_merge: Option, #[serde(default)] pub block_cache_size: Option, #[serde(default)] pub cache_index_and_filter_blocks: Option, #[serde(default)] #[serde(alias = "compression")] #[serde(deserialize_with = "to_rocksdb_compression_type_opt")] pub compression_type: Option, #[serde(default)] #[serde(alias = "wal_compression")] #[serde(deserialize_with = "to_rocksdb_compression_type_opt")] pub wal_compression_type: Option, #[serde(default)] pub wal_ttl_seconds: Option, #[serde(default)] pub wal_size_limit_mb: Option, #[serde(default)] pub wal_bytes_per_sync: Option, #[serde(default)] #[serde(deserialize_with = "to_rocksdb_recovery_mode_opt")] pub wal_recovery_mode: Option, #[serde(default)] pub compression_level: Option, #[serde(default)] #[serde(alias = "compression_start_level")] pub min_level_to_compress: Option, #[serde(default)] #[serde(alias = "level0_file_num_compaction_trigger")] pub level_zero_file_num_compaction_trigger: Option, #[serde(default)] #[serde(alias = "level0_slowdown_writes_trigger")] pub level_zero_slowdown_writes_trigger: Option, #[serde(default)] #[serde(alias = "level0_stop_writes_trigger")] pub level_zero_stop_writes_trigger: Option, #[serde(default)] pub max_bytes_for_level_base: Option, #[serde(default)] pub max_bytes_for_level_multiplier: Option, #[serde(default)] pub target_file_size_base: Option, #[serde(default)] pub max_background_jobs: Option, #[serde(default)] #[serde(alias = "max_compactions")] pub max_subcompactions: Option, #[serde(default)] pub max_flushes: Option, #[serde(default)] pub stats_dump_period_sec: Option, } fn default_write_buffer_size() -> Option { Some(16384) } fn parse_rocksdb_compression_type( str: &str, ) -> Result { // NOTE: Some values are not available because not compiled in rocksdb // (feature flag is off). match str.to_ascii_lowercase().as_str() { "none" => Ok(rocksdb::DBCompressionType::None), // "snappy" => Ok(rocksdb::DBCompressionType::Snappy), // "zlib" => Ok(rocksdb::DBCompressionType::Zlib), // "bz2" => Ok(rocksdb::DBCompressionType::Bz2), // "lz4" => Ok(rocksdb::DBCompressionType::Lz4), // "lz4hc" => Ok(rocksdb::DBCompressionType::Lz4hc), "zstd" => Ok(rocksdb::DBCompressionType::Zstd), _ => Err(serde::de::Error::unknown_variant( str, &[ "none", // "snappy", // "zlib", // "bz2", // "lz4", // "lz4hc", "zstd", ], )), } } fn to_rocksdb_compression_type_opt<'de, D>( deserializer: D, ) -> Result, D::Error> where D: serde::de::Deserializer<'de>, { let str: Option = Deserialize::deserialize(deserializer)?; str.map(|s| parse_rocksdb_compression_type(&s)).transpose() } fn parse_rocksdb_recovery_mode( str: &str, ) -> Result { match str.to_ascii_lowercase().as_str() { "tolerate_corrupted_tail_records" | "TolerateCorruptedTailRecords" => { Ok(rocksdb::DBRecoveryMode::TolerateCorruptedTailRecords) } "absolute_consistency" | "AbsoluteConsistency" => { Ok(rocksdb::DBRecoveryMode::AbsoluteConsistency) } "point_in_time" | "PointInTime" => Ok(rocksdb::DBRecoveryMode::PointInTime), "skip_any_corrupted_record" | "SkipAnyCorruptedRecord" => { Ok(rocksdb::DBRecoveryMode::SkipAnyCorruptedRecord) } _ => Err(serde::de::Error::unknown_variant( str, &[ "tolerate_corrupted_tail_records", "absolute_consistency", "point_in_time", "skip_any_corrupted_record", ], )), } } fn to_rocksdb_recovery_mode_opt<'de, D>( deserializer: D, ) -> Result, D::Error> where D: serde::de::Deserializer<'de>, { let str: Option = Deserialize::deserialize(deserializer)?; str.map(|s| parse_rocksdb_recovery_mode(&s)).transpose() } #[derive(Debug, Deserialize)] pub struct FstStoreConfig { #[serde(deserialize_with = "env_var::path_buf")] pub path: PathBuf, pub pool: FstStorePoolConfig, pub graph: FstStoreGraphConfig, } #[derive(Debug, Deserialize)] pub struct FstStorePoolConfig { pub inactive_after: u64, } #[derive(Debug, Deserialize)] pub struct FstStoreGraphConfig { pub consolidate_after: u64, pub max_size: usize, pub max_words: usize, } #[cfg(test)] pub(crate) mod tests { pub(crate) fn defaults_toml() -> &'static str { r#" [channel] inet = "[::1]:1491" tcp_timeout = 300 [normalization] unicode_normalization = "none" diacritic_folding_enabled = true stemming_enabled = false [tokenization] detect_special_patterns = true compat_split_special_patterns = false [stopwords] allow = [] deny = [] [search] query_limit_default = 10 query_limit_maximum = 100 query_alternates_try = 4 query_minimum_term_idf_default = 0.1 query_minimum_term_idf_minimum_object_count = 100 suggest_limit_default = 5 suggest_limit_maximum = 20 list_limit_default = 100 list_limit_maximum = 500 [store.kv] path = "./data/store/kv/" retain_word_objects = 1000 pool.inactive_after = 1800 database.flush_after = 900 database.compression_type = "zstd" database.parallelism = 2 database.write_ahead_log = true [store.fst] path = "./data/store/fst/" pool.inactive_after = 300 graph.consolidate_after = 180 graph.max_size = 2048 graph.max_words = 250000 "# } }