Upstream valeriansaliou/sonic state of the touched files before abd3d56935

core/src/config.rs

@@ -0,0 +1,407 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+//! Sonic library configuration.
+//!
+//! It does not include server nor channel configuration, which are specific
+//! to the `sonic-server` binary.
+
+use std::collections::HashSet;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use serde::Deserialize;
+
+use crate::util::serde::env_var;
+
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    pub normalization: NormalizationConfig,
+
+    pub tokenization: TokenizationConfig,
+
+    pub stopwords: StopwordsConfig,
+
+    pub search: SearchConfig,
+
+    pub store: StoreConfig,
+}
+
+impl Config {
+    pub fn validate(&self) {
+        // Check 'write_buffer' for KV
+        if self.store.kv.database.write_buffer_size == Some(0) {
+            panic!("write_buffer for kv must not be zero");
+        }
+
+        // Check 'flush_after' for KV
+        if self.store.kv.database.flush_after >= self.store.kv.pool.inactive_after {
+            panic!("flush_after for kv must be strictly lower than inactive_after");
+        }
+
+        // Check 'consolidate_after' for FST
+        if self.store.fst.graph.consolidate_after >= self.store.fst.pool.inactive_after {
+            panic!("consolidate_after for fst must be strictly lower than inactive_after");
+        }
+    }
+}
+
+/// Configuration group for normalization options (Unicode normalization,
+/// stemming, lemmatization…).
+#[derive(Debug, Deserialize, Clone, Copy)]
+pub struct NormalizationConfig {
+    #[serde(with = "crate::util::serde::none_string_as_none")]
+    pub unicode_normalization: Option<UnicodeNormalization>,
+
+    pub diacritic_folding_enabled: bool,
+
+    #[cfg(feature = "stemming")]
+    pub stemming_enabled: bool,
+}
+
+#[derive(Deserialize, Debug, Clone, Copy)]
+#[serde(rename_all = "snake_case")]
+pub enum UnicodeNormalization {
+    /// Unicode Normalization Form C.
+    Nfc,
+    /// Unicode Normalization Form KC.
+    Nfkc,
+}
+
+/// Configuration group for tokenization options.
+#[derive(Debug, Deserialize, Clone, Copy)]
+pub struct TokenizationConfig {
+    pub detect_special_patterns: bool,
+
+    #[serde(alias = "split_special_patterns")]
+    pub compat_split_special_patterns: bool,
+}
+
+#[derive(Debug, Deserialize, Clone, Default)]
+pub struct StopwordsConfig {
+    #[serde(deserialize_with = "to_stopwords")]
+    pub allow: HashSet<String>,
+
+    #[serde(deserialize_with = "to_stopwords")]
+    pub deny: HashSet<String>,
+}
+
+fn to_stopwords<'de, D>(deserializer: D) -> Result<HashSet<String>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    use unicode_normalization::UnicodeNormalization as _;
+
+    let vec: Vec<Box<str>> = Deserialize::deserialize(deserializer)?;
+    let stopwords_iter = vec.into_iter().map(|s| s.nfkd().to_string());
+    Ok(HashSet::from_iter(stopwords_iter))
+}
+
+#[derive(Debug, Deserialize)]
+pub struct SearchConfig {
+    pub query_limit_default: u16,
+
+    pub query_limit_maximum: u16,
+
+    pub query_alternates_try: usize,
+
+    pub query_minimum_term_idf_default: f32,
+
+    pub query_minimum_term_idf_minimum_object_count: u64,
+
+    pub suggest_limit_default: u16,
+
+    pub suggest_limit_maximum: u16,
+
+    pub list_limit_default: u16,
+
+    pub list_limit_maximum: u16,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct StoreConfig {
+    pub kv: Arc<KvStoreConfig>,
+
+    pub fst: Arc<FstStoreConfig>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct KvStoreConfig {
+    #[serde(deserialize_with = "env_var::path_buf")]
+    pub path: PathBuf,
+
+    pub retain_word_objects: usize,
+
+    pub pool: KvStorePoolConfig,
+
+    pub database: KvStoreDatabaseConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct KvStorePoolConfig {
+    pub inactive_after: u64,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct KvStoreDatabaseConfig {
+    pub flush_after: u64,
+
+    pub write_ahead_log: bool,
+
+    /// Whether or not to compress.
+    ///
+    /// Will get overriden if [`compression_type`](Self::compression_type) is
+    /// also specified.
+    #[serde(default)]
+    pub compress: Option<bool>,
+
+    #[serde(default)]
+    pub parallelism: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "max_files")]
+    pub max_open_files: Option<i32>,
+
+    // TODO(major): Make this MB, as in Kvrocks.
+    /// WARN: In KB!
+    #[serde(default = "default_write_buffer_size")]
+    #[serde(alias = "write_buffer")]
+    pub write_buffer_size: Option<usize>,
+
+    #[serde(default)]
+    pub max_write_buffer_number: Option<i32>,
+
+    #[serde(default)]
+    pub min_write_buffer_number: Option<i32>,
+
+    #[serde(default)]
+    pub min_write_buffer_number_to_merge: Option<i32>,
+
+    #[serde(default)]
+    pub block_cache_size: Option<u32>,
+
+    #[serde(default)]
+    pub cache_index_and_filter_blocks: Option<bool>,
+
+    #[serde(default)]
+    #[serde(alias = "compression")]
+    #[serde(deserialize_with = "to_rocksdb_compression_type_opt")]
+    pub compression_type: Option<rocksdb::DBCompressionType>,
+
+    #[serde(default)]
+    #[serde(alias = "wal_compression")]
+    #[serde(deserialize_with = "to_rocksdb_compression_type_opt")]
+    pub wal_compression_type: Option<rocksdb::DBCompressionType>,
+
+    #[serde(default)]
+    pub wal_ttl_seconds: Option<u64>,
+
+    #[serde(default)]
+    pub wal_size_limit_mb: Option<u64>,
+
+    #[serde(default)]
+    pub wal_bytes_per_sync: Option<u64>,
+
+    #[serde(default)]
+    #[serde(deserialize_with = "to_rocksdb_recovery_mode_opt")]
+    pub wal_recovery_mode: Option<rocksdb::DBRecoveryMode>,
+
+    #[serde(default)]
+    pub compression_level: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "compression_start_level")]
+    pub min_level_to_compress: Option<std::ffi::c_int>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_file_num_compaction_trigger")]
+    pub level_zero_file_num_compaction_trigger: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_slowdown_writes_trigger")]
+    pub level_zero_slowdown_writes_trigger: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_stop_writes_trigger")]
+    pub level_zero_stop_writes_trigger: Option<i32>,
+
+    #[serde(default)]
+    pub max_bytes_for_level_base: Option<u64>,
+
+    #[serde(default)]
+    pub max_bytes_for_level_multiplier: Option<f64>,
+
+    #[serde(default)]
+    pub target_file_size_base: Option<u64>,
+
+    #[serde(default)]
+    pub max_background_jobs: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "max_compactions")]
+    pub max_subcompactions: Option<u32>,
+
+    #[serde(default)]
+    pub max_flushes: Option<i32>,
+
+    #[serde(default)]
+    pub stats_dump_period_sec: Option<u32>,
+}
+
+fn default_write_buffer_size() -> Option<usize> {
+    Some(16384)
+}
+
+fn parse_rocksdb_compression_type<E: serde::de::Error>(
+    str: &str,
+) -> Result<rocksdb::DBCompressionType, E> {
+    // NOTE: Some values are not available because not compiled in rocksdb
+    //   (feature flag is off).
+    match str.to_ascii_lowercase().as_str() {
+        "none" => Ok(rocksdb::DBCompressionType::None),
+        // "snappy" => Ok(rocksdb::DBCompressionType::Snappy),
+        // "zlib" => Ok(rocksdb::DBCompressionType::Zlib),
+        // "bz2" => Ok(rocksdb::DBCompressionType::Bz2),
+        // "lz4" => Ok(rocksdb::DBCompressionType::Lz4),
+        // "lz4hc" => Ok(rocksdb::DBCompressionType::Lz4hc),
+        "zstd" => Ok(rocksdb::DBCompressionType::Zstd),
+        _ => Err(serde::de::Error::unknown_variant(
+            str,
+            &[
+                "none",
+                // "snappy",
+                // "zlib",
+                // "bz2",
+                // "lz4",
+                // "lz4hc",
+                "zstd",
+            ],
+        )),
+    }
+}
+
+fn to_rocksdb_compression_type_opt<'de, D>(
+    deserializer: D,
+) -> Result<Option<rocksdb::DBCompressionType>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    let str: Option<String> = Deserialize::deserialize(deserializer)?;
+    str.map(|s| parse_rocksdb_compression_type(&s)).transpose()
+}
+
+fn parse_rocksdb_recovery_mode<E: serde::de::Error>(
+    str: &str,
+) -> Result<rocksdb::DBRecoveryMode, E> {
+    match str.to_ascii_lowercase().as_str() {
+        "tolerate_corrupted_tail_records" | "TolerateCorruptedTailRecords" => {
+            Ok(rocksdb::DBRecoveryMode::TolerateCorruptedTailRecords)
+        }
+        "absolute_consistency" | "AbsoluteConsistency" => {
+            Ok(rocksdb::DBRecoveryMode::AbsoluteConsistency)
+        }
+        "point_in_time" | "PointInTime" => Ok(rocksdb::DBRecoveryMode::PointInTime),
+        "skip_any_corrupted_record" | "SkipAnyCorruptedRecord" => {
+            Ok(rocksdb::DBRecoveryMode::SkipAnyCorruptedRecord)
+        }
+        _ => Err(serde::de::Error::unknown_variant(
+            str,
+            &[
+                "tolerate_corrupted_tail_records",
+                "absolute_consistency",
+                "point_in_time",
+                "skip_any_corrupted_record",
+            ],
+        )),
+    }
+}
+
+fn to_rocksdb_recovery_mode_opt<'de, D>(
+    deserializer: D,
+) -> Result<Option<rocksdb::DBRecoveryMode>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    let str: Option<String> = Deserialize::deserialize(deserializer)?;
+    str.map(|s| parse_rocksdb_recovery_mode(&s)).transpose()
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStoreConfig {
+    #[serde(deserialize_with = "env_var::path_buf")]
+    pub path: PathBuf,
+
+    pub pool: FstStorePoolConfig,
+
+    pub graph: FstStoreGraphConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStorePoolConfig {
+    pub inactive_after: u64,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStoreGraphConfig {
+    pub consolidate_after: u64,
+
+    pub max_size: usize,
+
+    pub max_words: usize,
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    pub(crate) fn defaults_toml() -> &'static str {
+        r#"
+        [channel]
+        inet = "[::1]:1491"
+        tcp_timeout = 300
+
+        [normalization]
+        unicode_normalization = "none"
+        diacritic_folding_enabled = true
+        stemming_enabled = false
+
+        [tokenization]
+        detect_special_patterns = true
+        compat_split_special_patterns = false
+
+        [stopwords]
+        allow = []
+        deny = []
+
+        [search]
+        query_limit_default = 10
+        query_limit_maximum = 100
+        query_alternates_try = 4
+        query_minimum_term_idf_default = 0.1
+        query_minimum_term_idf_minimum_object_count = 100
+        suggest_limit_default = 5
+        suggest_limit_maximum = 20
+        list_limit_default = 100
+        list_limit_maximum = 500
+
+        [store.kv]
+        path = "./data/store/kv/"
+        retain_word_objects = 1000
+        pool.inactive_after = 1800
+        database.flush_after = 900
+        database.compression_type = "zstd"
+        database.parallelism = 2
+        database.write_ahead_log = true
+
+        [store.fst]
+        path = "./data/store/fst/"
+        pool.inactive_after = 300
+        graph.consolidate_after = 180
+        graph.max_size = 2048
+        graph.max_words = 250000
+        "#
+    }
+}