Sd rust tooling

.github/workflows/ci.yml

@@ -65,6 +65,10 @@ jobs:
       - run: cargo audit
       - run: cargo deny check
       - run: cargo machete
+      # Copy-paste detection (regression ratchet): fails if duplication in the crate sources
+      # exceeds the threshold in .jscpd.json. Node is preinstalled on ubuntu runners.
+      - name: Duplicate-code check (jscpd)
+        run: npx --yes jscpd sdsearch-core/src sdsearch-php/src
 
   coverage:
     name: Coverage (sdsearch-core -> octocov)

.jscpd.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://raw.githubusercontent.com/kucherenko/jscpd/master/packages/jscpd/schema.json",
+  "threshold": 7,
+  "minTokens": 50,
+  "reporters": ["console"],
+  "gitignore": true,
+  "absolute": true,
+  "ignore": ["**/target/**"]
+}

Cargo.toml

@@ -16,3 +16,58 @@ members = ["sdsearch-core", "sdsearch-php"]
 strip = "symbols"   # drop the symbol table: smaller binary, no internal-name leakage
 lto = "fat"         # whole-program LTO: maximum optimization (build time is not a concern)
 codegen-units = 1   # let the optimizer work across the whole crate
+
+# Workspace-wide lint policy, inherited by each member via `[lints] workspace = true`.
+# CI runs `cargo clippy --all-targets -- -D warnings`, so every `warn` here is effectively a
+# hard gate. Levels are curated to be high-signal on THIS codebase (an FFI extension with
+# unsafe + heavy hand-rolled byte parsing), not the full pedantic firehose.
+[workspace.lints.rust]
+# In an `unsafe fn`, still require an explicit `unsafe {}` around each unsafe op — the FFI /
+# mmap / GlobalAlloc code is exactly where a silently-unsafe body hides a real hazard.
+unsafe_op_in_unsafe_fn = "deny"
+# `pub` items not actually reachable from the crate's public API should be `pub(crate)`.
+unreachable_pub = "warn"
+
+[workspace.lints.clippy]
+# pedantic as a baseline (priority -1 so the specific `allow`s below win over it), then opt OUT
+# of the pedantic lints that are noise on THIS codebase instead of fixing hundreds of call sites.
+pedantic = { level = "warn", priority = -1 }
+
+# Intentional narrowing/among-format casts. This crate hand-decodes a binary index format where
+# `read_vint(...) as usize`, `len as u32`, etc. are deliberate and bounds-are-checked elsewhere;
+# rewriting every one as TryFrom would bloat and slow the hot parse paths for no real safety win.
+cast_possible_truncation = "allow"
+cast_precision_loss = "allow"
+cast_sign_loss = "allow"
+cast_possible_wrap = "allow"
+
+# Doc-prose nags with high false-positive rates on our comments (bare `.tis`, `ZSL`, `TermInfo`,
+# etc. flagged as needing backticks; every fallible/panicking internal fn asked for a section).
+doc_markdown = "allow"
+missing_errors_doc = "allow"
+missing_panics_doc = "allow"
+must_use_candidate = "allow"
+
+# Structural nags that fight the existing (deliberate) style of this codebase.
+too_many_lines = "allow"
+similar_names = "allow"
+items_after_statements = "allow"
+implicit_hasher = "allow"
+
+# Exact float compares live only in tests asserting known table values (encode_norm, length_norm,
+# a zero score) — an exact `==` is precisely what those want.
+float_cmp = "allow"
+# Segment file extensions are always written lowercase by our own writer, so the case-sensitive
+# `ends_with(".tis")` comparisons are correct and intentional.
+case_sensitive_file_extension_comparisons = "allow"
+# Consumer/builder APIs (e.g. `IndexReader::add_document(self, doc: Document)`) deliberately take
+# ownership even when the current body doesn't move every field — that is the intended contract.
+needless_pass_by_value = "allow"
+# `&Option<T>` is a fine signature for an internal helper; not worth rippling through call sites.
+ref_option = "allow"
+# JSON usage examples in doc comments (`{"fields":[...]}`) trip the intra-doc-link heuristic.
+doc_link_with_quotes = "allow"
+
+# leftover debugging that must never reach a release.
+todo = "warn"
+dbg_macro = "warn"

rustfmt.toml

@@ -0,0 +1,4 @@
+# Pin the formatting style explicitly so `cargo fmt --check` is deterministic across rustfmt
+# versions. Without this, the style follows rustfmt's floating default, which differs between
+# toolchain releases (e.g. import ordering changed under the 2024 style) and silently drifts CI.
+style_edition = "2024"

sdsearch-core/Cargo.toml

@@ -9,6 +9,9 @@ authors = ["InvGate"]
 keywords = ["lucene", "search", "zend", "php", "index"]
 categories = ["text-processing"]
 
+[lints]
+workspace = true
+
 [lib]
 name = "sdsearch_core"
 

sdsearch-core/examples/append_writer.rs

@@ -9,7 +9,7 @@
 //!       {"name":"title","value":"...","kind":"text","stored":true}, ... ] } ] }
 //!   kind ∈ {"text","keyword","unindexed"} (default stored=true)
 
-use sdsearch_core::zsl::writer::{append_documents, FieldKind, WriterDoc, WriterField, WriterOpts};
+use sdsearch_core::zsl::writer::{FieldKind, WriterDoc, WriterField, WriterOpts, append_documents};
 use std::path::Path;
 
 #[derive(serde::Deserialize)]
@@ -77,7 +77,11 @@ fn main() {
     // JSON output for the harness to parse (includes timing + process peak RSS)
     println!(
         "{{\"segment\":\"{}\",\"doc_count\":{},\"generation\":{},\"elapsed_ms\":{:.3},\"peak_rss_kb\":{}}}",
-        report.segment_name, report.doc_count, report.generation, elapsed_ms, peak_rss_kb()
+        report.segment_name,
+        report.doc_count,
+        report.generation,
+        elapsed_ms,
+        peak_rss_kb()
     );
 }
 

sdsearch-core/examples/bench_engine.rs

@@ -55,41 +55,45 @@ struct TrackingAlloc;
 
 unsafe impl GlobalAlloc for TrackingAlloc {
     unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
-        let p = System.alloc(layout);
-        if !p.is_null() && TRACK.load(Ordering::Relaxed) {
-            let live = LIVE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
-            PEAK.fetch_max(live, Ordering::Relaxed);
+        unsafe {
+            let p = System.alloc(layout);
+            if !p.is_null() && TRACK.load(Ordering::Relaxed) {
+                let live = LIVE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
+                PEAK.fetch_max(live, Ordering::Relaxed);
+            }
+            p
         }
-        p
     }
     unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
-        System.dealloc(ptr, layout);
-        if TRACK.load(Ordering::Relaxed) {
-            LIVE.fetch_sub(layout.size(), Ordering::Relaxed);
+        unsafe {
+            System.dealloc(ptr, layout);
+            if TRACK.load(Ordering::Relaxed) {
+                LIVE.fetch_sub(layout.size(), Ordering::Relaxed);
+            }
         }
     }
     unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
-        let p = System.realloc(ptr, layout, new_size);
-        if !p.is_null() && TRACK.load(Ordering::Relaxed) {
-            let old = layout.size();
-            if new_size >= old {
-                let live = LIVE.fetch_add(new_size - old, Ordering::Relaxed) + (new_size - old);
-                PEAK.fetch_max(live, Ordering::Relaxed);
-            } else {
-                LIVE.fetch_sub(old - new_size, Ordering::Relaxed);
+        unsafe {
+            let p = System.realloc(ptr, layout, new_size);
+            if !p.is_null() && TRACK.load(Ordering::Relaxed) {
+                let old = layout.size();
+                if new_size >= old {
+                    let live = LIVE.fetch_add(new_size - old, Ordering::Relaxed) + (new_size - old);
+                    PEAK.fetch_max(live, Ordering::Relaxed);
+                } else {
+                    LIVE.fetch_sub(old - new_size, Ordering::Relaxed);
+                }
             }
+            p
         }
-        p
     }
 }
 
 /// whether this process is the HEAP pass (TRACK on) or the TIME pass (TRACK off). Read once from
 /// `BENCH_TRACK_HEAP` (default: on). Startup allocations before this are minimal and precede any
 /// `reset_peak`, so LIVE accounting stays consistent for the whole measured region.
 fn init_track() -> bool {
-    let on = std::env::var("BENCH_TRACK_HEAP")
-        .map(|v| v != "0")
-        .unwrap_or(true);
+    let on = std::env::var("BENCH_TRACK_HEAP").map_or(true, |v| v != "0");
     TRACK.store(on, Ordering::Relaxed);
     on
 }
@@ -265,10 +269,16 @@ fn run_rebuild(n: usize, cap: usize) {
     let ms = t0.elapsed().as_secs_f64() * 1000.0;
     // total LIVE docs (base + added) via a post-commit reader — CommitReport.doc_count is the
     // per-session count, not the index total (see index_writer.rs commit_inner).
-    let doc_count = ZslIndex::open(&dir).map(|r| r.num_docs()).unwrap_or(0);
+    let doc_count = ZslIndex::open(&dir).map_or(0, |r| r.num_docs());
     println!(
         "{{\"engine\":\"native\",\"pass\":\"{}\",\"workload\":\"rebuild\",\"n\":{},\"cap\":{},\"ms\":{:.3},\"heap_peak_kb\":{},\"rss_peak_kb\":{},\"doc_count\":{}}}",
-        pass_label(), n, cap, ms, heap_peak_kb(), rss_peak_kb(), doc_count
+        pass_label(),
+        n,
+        cap,
+        ms,
+        heap_peak_kb(),
+        rss_peak_kb(),
+        doc_count
     );
     std::fs::remove_dir_all(&dir).ok();
 }
@@ -296,10 +306,18 @@ fn run_churn(n: usize, cap: usize) {
     }
     w.commit().expect("commit failed");
     let ms = t0.elapsed().as_secs_f64() * 1000.0;
-    let doc_count = ZslIndex::open(&dir).map(|r| r.num_docs()).unwrap_or(0);
+    let doc_count = ZslIndex::open(&dir).map_or(0, |r| r.num_docs());
     println!(
         "{{\"engine\":\"native\",\"pass\":\"{}\",\"workload\":\"churn\",\"n\":{},\"cap\":{},\"pct1\":{},\"ms\":{:.3},\"heap_peak_kb\":{},\"rss_peak_kb\":{},\"doc_count_before\":{},\"doc_count\":{}}}",
-        pass_label(), n, cap, one_pct, ms, heap_peak_kb(), rss_peak_kb(), doc_count_before, doc_count
+        pass_label(),
+        n,
+        cap,
+        one_pct,
+        ms,
+        heap_peak_kb(),
+        rss_peak_kb(),
+        doc_count_before,
+        doc_count
     );
     std::fs::remove_dir_all(&dir).ok();
 }
@@ -354,15 +372,20 @@ fn run_search(n: usize, iters: usize) {
     }
     println!(
         "{{\"engine\":\"native\",\"pass\":\"{}\",\"workload\":\"search\",\"n\":{},\"iters\":{},\"heap_peak_kb\":{},\"rss_peak_kb\":{},{}}}",
-        pass_label(), n, iters, heap_peak_kb(), rss_peak_kb(), parts.join(",")
+        pass_label(),
+        n,
+        iters,
+        heap_peak_kb(),
+        rss_peak_kb(),
+        parts.join(",")
     );
     std::fs::remove_dir_all(&dir).ok();
 }
 
 fn main() {
     init_track(); // TIME pass (BENCH_TRACK_HEAP=0) vs HEAP pass (default), before any measured op.
     let args: Vec<String> = std::env::args().collect();
-    let workload = args.get(1).map(String::as_str).unwrap_or("rebuild");
+    let workload = args.get(1).map_or("rebuild", String::as_str);
     let n: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1000);
     match workload {
         "rebuild" => run_rebuild(n, args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000)),

sdsearch-core/examples/diff_read.rs

@@ -6,7 +6,7 @@
 //!   queries.json: {"fields":["body","tag"],"queries":[{"name":"work","text":"work"},...]}
 
 use sdsearch_core::index::IndexReader;
-use sdsearch_core::query::{build_query, search, QueryParams};
+use sdsearch_core::query::{QueryParams, build_query, search};
 use sdsearch_core::zsl::index::ZslIndex;
 use std::collections::BTreeMap;
 use std::path::Path;

sdsearch-core/examples/optimize_bench.rs

@@ -36,29 +36,35 @@ struct TrackingAlloc;
 
 unsafe impl GlobalAlloc for TrackingAlloc {
     unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
-        let p = System.alloc(layout);
-        if !p.is_null() {
-            let live = LIVE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
-            PEAK.fetch_max(live, Ordering::Relaxed);
+        unsafe {
+            let p = System.alloc(layout);
+            if !p.is_null() {
+                let live = LIVE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
+                PEAK.fetch_max(live, Ordering::Relaxed);
+            }
+            p
         }
-        p
     }
     unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
-        System.dealloc(ptr, layout);
-        LIVE.fetch_sub(layout.size(), Ordering::Relaxed);
+        unsafe {
+            System.dealloc(ptr, layout);
+            LIVE.fetch_sub(layout.size(), Ordering::Relaxed);
+        }
     }
     unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
-        let p = System.realloc(ptr, layout, new_size);
-        if !p.is_null() {
-            let old = layout.size();
-            if new_size >= old {
-                let live = LIVE.fetch_add(new_size - old, Ordering::Relaxed) + (new_size - old);
-                PEAK.fetch_max(live, Ordering::Relaxed);
-            } else {
-                LIVE.fetch_sub(old - new_size, Ordering::Relaxed);
+        unsafe {
+            let p = System.realloc(ptr, layout, new_size);
+            if !p.is_null() {
+                let old = layout.size();
+                if new_size >= old {
+                    let live = LIVE.fetch_add(new_size - old, Ordering::Relaxed) + (new_size - old);
+                    PEAK.fetch_max(live, Ordering::Relaxed);
+                } else {
+                    LIVE.fetch_sub(old - new_size, Ordering::Relaxed);
+                }
             }
+            p
         }
-        p
     }
 }
 
@@ -265,7 +271,13 @@ fn run_existing(args: &[String]) {
 
     println!(
         "{{\"source\":\"{}\",\"n_extra\":{},\"segments_before\":{},\"doc_count\":{},\"optimize_ms\":{:.2},\"heap_peak_kb\":{},\"vmhwm_kb\":{}}}",
-        src.display(), n_extra, segments_before, report.doc_count, optimize_ms, heap_peak_kb, vmhwm_kb()
+        src.display(),
+        n_extra,
+        segments_before,
+        report.doc_count,
+        optimize_ms,
+        heap_peak_kb,
+        vmhwm_kb()
     );
 
     std::fs::remove_dir_all(scratch).ok();
@@ -306,7 +318,13 @@ fn run_hot(args: &[String]) {
 
     println!(
         "{{\"mode\":\"hot\",\"n\":{},\"cap\":{},\"segments_before\":{},\"doc_count\":{},\"optimize_ms\":{:.2},\"heap_peak_kb\":{},\"vmhwm_kb\":{}}}",
-        n, cap, segments_before, report.doc_count, optimize_ms, heap_peak_kb, vmhwm_kb()
+        n,
+        cap,
+        segments_before,
+        report.doc_count,
+        optimize_ms,
+        heap_peak_kb,
+        vmhwm_kb()
     );
 
     std::fs::remove_dir_all(&dir).ok();
@@ -354,7 +372,13 @@ fn main() {
 
     println!(
         "{{\"n\":{},\"cap\":{},\"segments_before\":{},\"doc_count\":{},\"optimize_ms\":{:.2},\"heap_peak_kb\":{},\"vmhwm_kb\":{}}}",
-        n, cap, segments_before, report.doc_count, optimize_ms, heap_peak_kb, vmhwm_kb()
+        n,
+        cap,
+        segments_before,
+        report.doc_count,
+        optimize_ms,
+        heap_peak_kb,
+        vmhwm_kb()
     );
 
     std::fs::remove_dir_all(&dir).ok();

sdsearch-core/examples/perf_native.rs

@@ -1,7 +1,7 @@
 //! perf of the native reader over a large user-provided ZSL index.
 //! Usage: SDSEARCH_PERF_INDEX=/path/to/index cargo run -p sdsearch-core --release --example perf_native
 //! If the env var is unset, exits 0 without running (for CI/dev without the index).
-use sdsearch_core::query::{build_query, search, QueryParams};
+use sdsearch_core::query::{QueryParams, build_query, search};
 use sdsearch_core::zsl::index::ZslIndex;
 use std::time::Instant;
 

sdsearch-core/src/analysis.rs

@@ -2,11 +2,10 @@
 //! tokenization regex deviates from stock Zend Lucene: tokens keep - _ . : # / @
 //! (emails, URLs, and ticket refs stay as a single term).
 
-use once_cell::sync::Lazy;
 use regex::Regex;
 
-static TOKEN_RE: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r"[\p{L}\p{N}\-_.:#/@]+[\p{L}\p{N}]?").unwrap());
+static TOKEN_RE: std::sync::LazyLock<Regex> =
+    std::sync::LazyLock::new(|| Regex::new(r"[\p{L}\p{N}\-_.:#/@]+[\p{L}\p{N}]?").unwrap());
 
 /// tokenizes and lowercases, replicating the legacy analyzer
 pub fn analyze(text: &str) -> Vec<String> {

sdsearch-core/src/distance.rs

@@ -16,7 +16,7 @@ pub fn levenshtein_bytes(a: &[u8], b: &[u8]) -> usize {
     for i in 1..=n {
         cur[0] = i;
         for j in 1..=m {
-            let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 };
+            let cost = usize::from(a[i - 1] != b[j - 1]);
             cur[j] = (prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + cost);
         }
         std::mem::swap(&mut prev, &mut cur);