diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 295c05c..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: build -on: [push, pull_request] - -jobs: - linux-nts: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: shivammathur/setup-php@v2 - with: - php-version: '8.4' - # setup-php provides NTS builds by default on Linux - # ext-php-rs uses bindgen, which needs libclang + clang's resource headers - # (not just the shared libclang-N library); without this, cargo build fails with - # "stddef.h file not found" when generating the PHP bindings. - - run: sudo apt-get update && sudo apt-get install -y clang libclang-dev - - run: cargo test -p sdsearch-core - - run: cargo build -p sdsearch-php --release - - run: php -d extension=$(pwd)/target/release/libsdsearch.so tests/smoke.php - - windows-nts: - runs-on: windows-latest - # ext-php-rs needs the vectorcall ABI on Windows (#![feature(abi_vectorcall)]), - # which is nightly-only. The repo's rust-toolchain.toml pins stable and would - # otherwise override the nightly toolchain installed below; RUSTUP_TOOLCHAIN - # takes precedence over rust-toolchain.toml, so it wins here. - env: - RUSTUP_TOOLCHAIN: nightly - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@nightly # Windows requires nightly (abi_vectorcall) - - uses: shivammathur/setup-php@v2 - with: - php-version: '8.4' # NTS x64 build from windows.php.net - # same bindgen requirement as linux-nts: libclang + clang resource headers - - uses: KyleMayes/install-llvm-action@v2 - with: - version: "18.1" - - run: cargo test -p sdsearch-core - - run: cargo build -p sdsearch-php --release - - run: php -d extension=${{ github.workspace }}\target\release\sdsearch.dll tests\smoke.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a3b764c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,151 @@ +name: ci + +# Gates (test, lint) run first and BLOCK the build jobs (needs:). The coverage job is +# informational (continue-on-error, in no job's needs) so a coverage/token hiccup never +# reddens the pipeline. Windows uses nightly Rust (ext-php-rs abi_vectorcall). +on: + push: + pull_request: + workflow_dispatch: {} + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + name: Rust tests + clippy (stable, PHP 8.4 NTS) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # cargo test / clippy --all-targets compile sdsearch-php, whose ext-php-rs bindgen step + # links against PHP internals — so PHP dev headers + clang are required even to test. + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + tools: none + extensions: none + - name: Assert PHP build is NTS + run: php -v | grep -q 'NTS' || { echo "::error::expected an NTS PHP build, got:"; php -v; exit 1; } + - name: Install clang (ext-php-rs bindgen) + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - run: cargo test + - run: cargo clippy --all-targets -- -D warnings + + lint: + name: Lint & supply-chain (fmt, audit, deny, machete) + runs-on: ubuntu-latest + # Lightweight: fmt reads source; audit/deny read Cargo.lock; machete parses manifests. + # None need clang or PHP (clippy runs against a real compile in the `test` job). + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - run: cargo fmt --all --check + - name: Install cargo-audit, cargo-deny, cargo-machete + uses: taiki-e/install-action@v2 + with: + tool: cargo-audit,cargo-deny,cargo-machete + - run: cargo audit + - run: cargo deny check + - run: cargo machete + + coverage: + name: Coverage (sdsearch-core -> octocov) + runs-on: ubuntu-latest + # Informational: never blocks the pipeline and is in no job's `needs`. sdsearch-core has no + # PHP dependency, so this job needs neither clang nor PHP. + continue-on-error: true + permissions: + contents: write # octocov commits the coverage badge to main + pull-requests: write # octocov posts the PR comment + actions: read # octocov reads the artifact datastore + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + - uses: Swatinem/rust-cache@v2 + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov + # Skip the interprocess-lock test: `cargo llvm-cov` runs `cargo test --tests`, which does not + # build the stream_writer example that test needs, so it fails only under llvm-cov. + - name: Measure coverage (sdsearch-core) + run: | + mkdir -p coverage + cargo llvm-cov -p sdsearch-core --lcov --output-path coverage/lcov.info \ + -- --skip write_lock_excludes_a_second_process + - name: octocov (PR comment + main badge) + uses: k1LoW/octocov-action@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + build-linux: + name: Build + smoke (Linux, PHP 8.4 NTS) + runs-on: ubuntu-latest + needs: [test, lint] + steps: + - uses: actions/checkout@v4 + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + tools: none + extensions: none + - name: Assert PHP build is NTS + run: php -v | grep -q 'NTS' || { echo "::error::expected an NTS PHP build, got:"; php -v; exit 1; } + - name: Install clang (ext-php-rs bindgen) + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo build -p sdsearch-php --release + - name: Smoke test (extension loads + sdsearch_version) + run: php -d extension="$(pwd)/target/release/libsdsearch.so" tests/smoke.php + - name: Stage Linux artifact + run: | + mkdir -p dist/linux + cp target/release/libsdsearch.so dist/linux/sdsearch.so + - uses: actions/upload-artifact@v4 + with: + name: sdsearch-linux-x64-php84nts + path: dist/linux/* + if-no-files-found: error + + build-windows: + name: Build + smoke (Windows, nightly, PHP 8.4 NTS) + runs-on: windows-latest + needs: [test, lint] + # RUSTUP_TOOLCHAIN takes precedence over rust-toolchain.toml (which pins stable); ext-php-rs + # needs nightly on Windows for abi_vectorcall. + env: + RUSTUP_TOOLCHAIN: nightly + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + - uses: KyleMayes/install-llvm-action@v2 + with: + version: "18.1" + - run: cargo build -p sdsearch-php --release + - name: Smoke test (extension loads + sdsearch_version) + run: php -d extension="${{ github.workspace }}\target\release\sdsearch.dll" tests\smoke.php + - name: Stage Windows artifact + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path dist\windows | Out-Null + Copy-Item target\release\sdsearch.dll dist\windows\sdsearch.dll + - uses: actions/upload-artifact@v4 + with: + name: sdsearch-windows-x64-php84nts + path: dist/windows/* + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 6e00f7c..4b4a6df 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /.superpowers/ /.remember/ /.claude/ +/coverage diff --git a/.octocov.yml b/.octocov.yml new file mode 100644 index 0000000..93b58ee --- /dev/null +++ b/.octocov.yml @@ -0,0 +1,29 @@ +# octocov: self-hosted coverage reporting (no external SaaS). +# - On PRs: posts/updates a comment with total + per-file coverage and the delta vs main. +# - On main: stores the report to a GitHub Actions artifact datastore and commits the badge. +coverage: + paths: + - coverage/lcov.info + badge: + path: docs/coverage.svg + +# Commit the regenerated badge back to main. "[skip ci]" prevents the commit from re-triggering CI. +push: + if: is_default_branch + message: "docs: update coverage badge [skip ci]" + +# Store the current report on main so PRs can diff against it. artifact:// = native Actions artifact, +# no branch, no external service. +report: + if: is_default_branch + datastores: + - artifact://${GITHUB_REPOSITORY} + +# On PRs, read the stored main report to compute the delta. +diff: + if: is_pull_request + datastores: + - artifact://${GITHUB_REPOSITORY} + +comment: + if: is_pull_request diff --git a/README.md b/README.md index b509d16..1da673c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # sdsearch -[![build](https://github.com/InvGate/php-sdsearch/actions/workflows/build.yml/badge.svg)](https://github.com/InvGate/php-sdsearch/actions/workflows/build.yml) +[![CI](https://github.com/InvGate/php-sdsearch/actions/workflows/ci.yml/badge.svg)](https://github.com/InvGate/php-sdsearch/actions/workflows/ci.yml) +[![coverage](https://raw.githubusercontent.com/InvGate/php-sdsearch/main/docs/coverage.svg)](https://github.com/InvGate/php-sdsearch/actions/workflows/ci.yml) [![license](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) A native Rust engine that reads **and** writes the [Zend Search Lucene](https://framework.zend.com/manual/1.12/en/zend.search.lucene.html) @@ -101,10 +102,14 @@ them as a shape-of-the-win indicator, not a guarantee. ## Continuous integration -`.github/workflows/build.yml` builds and tests on both **Linux** and **Windows** on every -push/PR: `cargo test -p sdsearch-core`, a release build of the PHP extension, and a smoke -test that loads it into PHP and calls into it. No ZSL install is needed in CI — it only -exercises the fixture-backed test suite and the extension build/load. +The pipeline in `.github/workflows/ci.yml` runs on every push/PR: + +- **`test`** — `cargo test` (whole workspace) and `cargo clippy --all-targets -- -D warnings`. +- **`lint`** — `cargo fmt --all --check` plus supply-chain checks: `cargo audit`, `cargo deny`, and `cargo machete`. +- **`coverage`** — measures `sdsearch-core` coverage with `cargo-llvm-cov` and reports it via [octocov](https://github.com/k1LoW/octocov): a comment on each PR with the coverage delta vs `main`, plus the coverage badge above. It is informational and never blocks the pipeline. +- **`build-linux` / `build-windows`** — release-build the PHP extension and run a smoke test that loads it into PHP, then upload the `.so`/`.dll` as build artifacts. These run only after `test` and `lint` pass. + +No ZSL install is needed in CI — it only exercises the fixture-backed test suite and the extension build/load. ## License diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..778177f --- /dev/null +++ b/deny.toml @@ -0,0 +1,35 @@ +# cargo-deny policy for sdsearch (Apache-2.0 project). +# License allow-list derived from `cargo deny list` over the dependency tree. +[graph] +all-features = true + +[advisories] +version = 2 +yanked = "deny" + +[licenses] +version = 2 +# Permissive licenses present across the dependency tree (all verified present via `cargo deny list`). +allow = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "MIT", + "MIT-0", + "BSD-3-Clause", + "ISC", + "Zlib", + "Unicode-3.0", + "CC0-1.0", + "CDLA-Permissive-2.0", + "Unlicense", + "bzip2-1.0.6", +] +confidence-threshold = 0.9 + +[bans] +# The dependency tree carries a few duplicated transitive versions (e.g. shlex); not worth failing CI over. +multiple-versions = "allow" + +[sources] +unknown-registry = "deny" +unknown-git = "deny" diff --git a/sdsearch-core/examples/append_writer.rs b/sdsearch-core/examples/append_writer.rs index 6938ec8..5995650 100644 --- a/sdsearch-core/examples/append_writer.rs +++ b/sdsearch-core/examples/append_writer.rs @@ -34,8 +34,12 @@ fn default_true() -> bool { fn main() { let args: Vec = std::env::args().collect(); - let index_dir = args.get(1).expect("usage: append_writer "); - let docs_path = args.get(2).expect("usage: append_writer "); + let index_dir = args + .get(1) + .expect("usage: append_writer "); + let docs_path = args + .get(2) + .expect("usage: append_writer "); let raw = std::fs::read_to_string(docs_path).expect("could not read docs.json"); let spec: Spec = serde_json::from_str(&raw).expect("invalid docs.json"); @@ -54,7 +58,12 @@ fn main() { "unindexed" => FieldKind::UnIndexed, other => panic!("unknown kind: {other}"), }; - WriterField { name: f.name, value: f.value, kind, stored: f.stored } + WriterField { + name: f.name, + value: f.value, + kind, + stored: f.stored, + } }) .collect(), }) diff --git a/sdsearch-core/examples/diff_read.rs b/sdsearch-core/examples/diff_read.rs index 5ef8230..d209a79 100644 --- a/sdsearch-core/examples/diff_read.rs +++ b/sdsearch-core/examples/diff_read.rs @@ -35,8 +35,12 @@ fn params(text: &str) -> QueryParams { fn main() { let args: Vec = std::env::args().collect(); - let dir = args.get(1).expect("usage: diff_read "); - let qpath = args.get(2).expect("usage: diff_read "); + let dir = args + .get(1) + .expect("usage: diff_read "); + let qpath = args + .get(2) + .expect("usage: diff_read "); let raw = std::fs::read_to_string(qpath).expect("could not read queries.json"); let spec: QSpec = serde_json::from_str(&raw).expect("invalid queries.json"); diff --git a/sdsearch-core/examples/stream_writer.rs b/sdsearch-core/examples/stream_writer.rs index 25f1716..f35ec39 100644 --- a/sdsearch-core/examples/stream_writer.rs +++ b/sdsearch-core/examples/stream_writer.rs @@ -69,14 +69,19 @@ fn main() { /// `stream_writer delete `: opens the `IndexWriter`, marks each gid /// as deleted (`delete_document`) and commits. Prints `{"deleted":N,"generation":G}`. fn run_delete(args: &[String]) { - let index_dir = args.get(2).expect("usage: stream_writer delete "); - let gids_raw = args.get(3).expect("usage: stream_writer delete "); + let index_dir = args + .get(2) + .expect("usage: stream_writer delete "); + let gids_raw = args + .get(3) + .expect("usage: stream_writer delete "); let gids: Vec = gids_raw .split(',') .map(|s| s.trim().parse().expect("invalid gid (expected an integer)")) .collect(); - let mut w = IndexWriter::open(Path::new(index_dir), WriterOpts::default()).expect("open failed"); + let mut w = + IndexWriter::open(Path::new(index_dir), WriterOpts::default()).expect("open failed"); for gid in &gids { w.delete_document(*gid); } @@ -92,7 +97,9 @@ fn run_delete(args: &[String]) { /// `stream_writer optimize `: opens the IndexWriter and runs optimize() (merge /// everything into 1 compacted segment). Prints `{"optimized":true,"generation":G,"doc_count":D}`. fn run_optimize(args: &[String]) { - let index_dir = args.get(2).expect("usage: stream_writer optimize "); + let index_dir = args + .get(2) + .expect("usage: stream_writer optimize "); let w = IndexWriter::open(Path::new(index_dir), WriterOpts::default()).expect("open failed"); let report = w.optimize().expect("optimize failed"); println!( @@ -106,7 +113,9 @@ fn run_optimize(args: &[String]) { /// LOCK_WOULDBLOCK and exits 3. Used by the inter-process test zsl_interprocess_lock.rs. fn run_hold_lock(args: &[String]) { use std::io::Write; - let index_dir = args.get(2).expect("usage: stream_writer hold-lock "); + let index_dir = args + .get(2) + .expect("usage: stream_writer hold-lock "); let hold_ms: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(0); match IndexWriter::open(Path::new(index_dir), WriterOpts::default()) { Ok(w) => { @@ -127,14 +136,21 @@ fn run_hold_lock(args: &[String]) { } fn run_stream(args: &[String]) { - let index_dir = args.get(1).expect("usage: stream_writer [cap]"); - let docs_path = args.get(2).expect("usage: stream_writer [cap]"); + let index_dir = args + .get(1) + .expect("usage: stream_writer [cap]"); + let docs_path = args + .get(2) + .expect("usage: stream_writer [cap]"); let cap: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(1000); let raw = std::fs::read_to_string(docs_path).expect("could not read docs.json"); let spec: Spec = serde_json::from_str(&raw).expect("invalid docs.json"); - let opts = WriterOpts { max_buffered_docs: cap, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: cap, + ..WriterOpts::default() + }; let t0 = std::time::Instant::now(); let mut w = IndexWriter::open(Path::new(index_dir), opts).expect("open failed"); for d in spec.docs { @@ -148,10 +164,16 @@ fn run_stream(args: &[String]) { "unindexed" => FieldKind::UnIndexed, other => panic!("unknown kind: {other}"), }; - WriterField { name: f.name, value: f.value, kind, stored: f.stored } + WriterField { + name: f.name, + value: f.value, + kind, + stored: f.stored, + } }) .collect(); - w.add_document(WriterDoc { fields }).expect("add_document failed"); + w.add_document(WriterDoc { fields }) + .expect("add_document failed"); } let report = w.commit().expect("commit failed"); let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0; diff --git a/sdsearch-core/src/analysis.rs b/sdsearch-core/src/analysis.rs index 6e02269..5e58bb3 100644 --- a/sdsearch-core/src/analysis.rs +++ b/sdsearch-core/src/analysis.rs @@ -33,11 +33,11 @@ mod tests { #[test] fn keeps_email_and_url_as_one_token() { - assert_eq!(analyze("Mail user@example.com"), vec!["mail", "user@example.com"]); assert_eq!( - analyze("see https://a.b/c"), - vec!["see", "https://a.b/c"] + analyze("Mail user@example.com"), + vec!["mail", "user@example.com"] ); + assert_eq!(analyze("see https://a.b/c"), vec!["see", "https://a.b/c"]); } #[test] diff --git a/sdsearch-core/src/index.rs b/sdsearch-core/src/index.rs index 55bbda7..ee1ec19 100644 --- a/sdsearch-core/src/index.rs +++ b/sdsearch-core/src/index.rs @@ -135,8 +135,10 @@ impl MemoryIndex { let mut builder = fst::MapBuilder::memory(); for (term, postings) in terms { let offset = postings_bin.len() as u64; - let mut docs: Vec<(usize, &Vec)> = - postings.iter().map(|(d, positions)| (*d, positions)).collect(); + let mut docs: Vec<(usize, &Vec)> = postings + .iter() + .map(|(d, positions)| (*d, positions)) + .collect(); docs.sort_by_key(|(d, _)| *d); write_vint(&mut postings_bin, docs.len() as u64); let mut prev = 0usize; @@ -166,8 +168,9 @@ impl MemoryIndex { } lengths.insert(field.clone(), v); } - let stored: Vec> = - (0..self.num_docs).map(|d| IndexReader::stored_fields(self, d)).collect(); + let stored: Vec> = (0..self.num_docs) + .map(|d| IndexReader::stored_fields(self, d)) + .collect(); let meta = SegmentMeta { format_version: 2, @@ -175,9 +178,18 @@ impl MemoryIndex { fields: field_names, }; let json_err = |e: serde_json::Error| std::io::Error::other(e); - std::fs::write(dir.join("meta.json"), serde_json::to_vec(&meta).map_err(json_err)?)?; - std::fs::write(dir.join("lengths.json"), serde_json::to_vec(&lengths).map_err(json_err)?)?; - std::fs::write(dir.join("stored.json"), serde_json::to_vec(&stored).map_err(json_err)?)?; + std::fs::write( + dir.join("meta.json"), + serde_json::to_vec(&meta).map_err(json_err)?, + )?; + std::fs::write( + dir.join("lengths.json"), + serde_json::to_vec(&lengths).map_err(json_err)?, + )?; + std::fs::write( + dir.join("stored.json"), + serde_json::to_vec(&stored).map_err(json_err)?, + )?; Ok(()) } } @@ -199,8 +211,10 @@ impl IndexReader for MemoryIndex { self.postings .get(&(field.to_string(), term.to_string())) .map(|p| { - let mut v: Vec<(usize, u32)> = - p.iter().map(|(d, positions)| (*d, positions.len() as u32)).collect(); + let mut v: Vec<(usize, u32)> = p + .iter() + .map(|(d, positions)| (*d, positions.len() as u32)) + .collect(); v.sort_by_key(|(d, _)| *d); v }) @@ -301,6 +315,9 @@ mod tests { d.add("title", "hello world", FieldKind::Text); d.add("body", "hello", FieldKind::Text); idx.add_document(d); - assert_eq!(idx.indexed_fields(), vec!["body".to_string(), "title".to_string()]); + assert_eq!( + idx.indexed_fields(), + vec!["body".to_string(), "title".to_string()] + ); } } diff --git a/sdsearch-core/src/query.rs b/sdsearch-core/src/query.rs index 54422cc..a5d34b9 100644 --- a/sdsearch-core/src/query.rs +++ b/sdsearch-core/src/query.rs @@ -2,7 +2,9 @@ //! and build_query: a port of Zend Lucene's boolean query builder for the surface the host application uses. use crate::index::IndexReader; -use crate::search::{finalize, fuzzy_terms, phrase_scores, term_scores, union_scores, wildcard_terms, Hit}; +use crate::search::{ + finalize, fuzzy_terms, phrase_scores, term_scores, union_scores, wildcard_terms, Hit, +}; use std::collections::HashMap; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -15,11 +17,28 @@ pub enum Occur { #[derive(Debug, Clone)] pub enum Query { /// exact term; field None = all indexed fields (ZSL null-field rewrite). - Term { field: Option, text: String }, - Wildcard { field: Option, pattern: String, min_prefix_len: usize }, - Fuzzy { field: Option, text: String, similarity: f32, prefix_len: usize }, - Phrase { field: String, terms: Vec }, - Boolean { clauses: Vec<(Occur, Query)> }, + Term { + field: Option, + text: String, + }, + Wildcard { + field: Option, + pattern: String, + min_prefix_len: usize, + }, + Fuzzy { + field: Option, + text: String, + similarity: f32, + prefix_len: usize, + }, + Phrase { + field: String, + terms: Vec, + }, + Boolean { + clauses: Vec<(Occur, Query)>, + }, } /// target fields of a leaf: the given one, or all indexed fields if None. @@ -42,7 +61,11 @@ fn eval(index: &impl IndexReader, q: &Query) -> HashMap { } acc } - Query::Wildcard { field, pattern, min_prefix_len } => { + Query::Wildcard { + field, + pattern, + min_prefix_len, + } => { let mut acc: HashMap = HashMap::new(); for f in target_fields(index, field) { let terms = wildcard_terms(index, &f, pattern, *min_prefix_len); @@ -53,7 +76,12 @@ fn eval(index: &impl IndexReader, q: &Query) -> HashMap { } acc } - Query::Fuzzy { field, text, similarity, prefix_len } => { + Query::Fuzzy { + field, + text, + similarity, + prefix_len, + } => { let mut acc: HashMap = HashMap::new(); for f in target_fields(index, field) { let terms = fuzzy_terms(index, &f, text, *similarity, *prefix_len); @@ -75,12 +103,21 @@ fn eval(index: &impl IndexReader, q: &Query) -> HashMap { /// Lucene-style boolean semantics: must (intersection, required), should (sum/coord), /// must-not (exclusion); if there is no must, at least one should must match. fn eval_boolean(index: &impl IndexReader, clauses: &[(Occur, Query)]) -> HashMap { - let musts: Vec> = - clauses.iter().filter(|(o, _)| *o == Occur::Must).map(|(_, q)| eval(index, q)).collect(); - let shoulds: Vec> = - clauses.iter().filter(|(o, _)| *o == Occur::Should).map(|(_, q)| eval(index, q)).collect(); - let mustnots: Vec> = - clauses.iter().filter(|(o, _)| *o == Occur::MustNot).map(|(_, q)| eval(index, q)).collect(); + let musts: Vec> = clauses + .iter() + .filter(|(o, _)| *o == Occur::Must) + .map(|(_, q)| eval(index, q)) + .collect(); + let shoulds: Vec> = clauses + .iter() + .filter(|(o, _)| *o == Occur::Should) + .map(|(_, q)| eval(index, q)) + .collect(); + let mustnots: Vec> = clauses + .iter() + .filter(|(o, _)| *o == Occur::MustNot) + .map(|(_, q)| eval(index, q)) + .collect(); // accumulated score and matched (should+must) clauses per doc, for the coord. let mut score: HashMap = HashMap::new(); @@ -128,7 +165,12 @@ fn eval_boolean(index: &impl IndexReader, clauses: &[(Occur, Query)]) -> HashMap let total = (musts.len() + shoulds.len()).max(1) as f32; score .into_iter() - .map(|(d, s)| (d, s * (matched.get(&d).copied().unwrap_or(0) as f32 / total))) + .map(|(d, s)| { + ( + d, + s * (matched.get(&d).copied().unwrap_or(0) as f32 / total), + ) + }) .collect() } @@ -202,36 +244,64 @@ impl std::error::Error for QueryError {} fn text_subquery(p: &QueryParams) -> Query { let lc = p.text.to_lowercase(); // escaping mirrors the host's query builder: str_replace ':', ',', '-' -> '\:', '\,', '\-'. - let esc = lc.replace(':', "\\:").replace(',', "\\,").replace('-', "\\-"); + let esc = lc + .replace(':', "\\:") + .replace(',', "\\,") + .replace('-', "\\-"); let esc_words: Vec<&str> = esc.split_whitespace().collect(); let mut clauses: Vec<(Occur, Query)> = Vec::new(); if esc_words.len() > 1 { for w in &esc_words { - clauses.push((Occur::Should, Query::Fuzzy { - field: None, text: (*w).to_string(), - similarity: p.fuzzy_similarity, prefix_len: p.fuzzy_prefix_len, - })); + clauses.push(( + Occur::Should, + Query::Fuzzy { + field: None, + text: (*w).to_string(), + similarity: p.fuzzy_similarity, + prefix_len: p.fuzzy_prefix_len, + }, + )); } } - clauses.push((Occur::Should, Query::Fuzzy { - field: None, text: esc.clone(), - similarity: p.fuzzy_similarity, prefix_len: p.fuzzy_prefix_len, - })); - clauses.push((Occur::Should, Query::Wildcard { - field: None, pattern: format!("{esc}*"), min_prefix_len: p.wildcard_min_prefix, - })); + clauses.push(( + Occur::Should, + Query::Fuzzy { + field: None, + text: esc.clone(), + similarity: p.fuzzy_similarity, + prefix_len: p.fuzzy_prefix_len, + }, + )); + clauses.push(( + Occur::Should, + Query::Wildcard { + field: None, + pattern: format!("{esc}*"), + min_prefix_len: p.wildcard_min_prefix, + }, + )); // QueryParser::parse(RAW text): the analyzer tokenizes the original text -> // one all-fields term (default-OR) per token. for tok in crate::analysis::analyze(&p.text) { - clauses.push((Occur::Should, Query::Term { field: None, text: tok })); + clauses.push(( + Occur::Should, + Query::Term { + field: None, + text: tok, + }, + )); } Query::Boolean { clauses } } /// key-field name (for IN): appends "_key" only if the field does not already contain it. fn key_field_in(field: &str) -> String { - if field.contains("_key") { field.to_string() } else { format!("{field}_key") } + if field.contains("_key") { + field.to_string() + } else { + format!("{field}_key") + } } /// builds the boolean Query equivalent to Zend Lucene's boolean query builder for the supported surface. @@ -256,7 +326,15 @@ pub fn build_query(p: &QueryParams) -> Result { let clauses: Vec<(Occur, Query)> = wg .values .iter() - .map(|v| (Occur::Should, Query::Term { field: Some(field.clone()), text: v.clone() })) + .map(|v| { + ( + Occur::Should, + Query::Term { + field: Some(field.clone()), + text: v.clone(), + }, + ) + }) .collect(); top.push((wg.occur, Query::Boolean { clauses })); } @@ -274,11 +352,22 @@ pub fn build_query(p: &QueryParams) -> Result { // IN: conditional key-field naming (appends "_key" only when missing). let field = key_field_in(&ig.field); for v in &ig.values { - in_clauses.push((Occur::Should, Query::Term { field: Some(field.clone()), text: v.clone() })); + in_clauses.push(( + Occur::Should, + Query::Term { + field: Some(field.clone()), + text: v.clone(), + }, + )); } } if !in_clauses.is_empty() { - top.push((Occur::Must, Query::Boolean { clauses: in_clauses })); + top.push(( + Occur::Must, + Query::Boolean { + clauses: in_clauses, + }, + )); } Ok(Query::Boolean { clauses: top }) @@ -294,7 +383,11 @@ mod tests { // doc0 title="vpn guide" lang="es"; doc1 title="vpn setup" lang="en"; // doc2 title="mysql notes" lang="es" let mut idx = MemoryIndex::new(); - let rows = [("vpn guide", "es"), ("vpn setup", "en"), ("mysql notes", "es")]; + let rows = [ + ("vpn guide", "es"), + ("vpn setup", "en"), + ("mysql notes", "es"), + ]; for (title, lang) in rows { let mut d = Document::new(); d.add("title", title, FieldKind::Text); @@ -315,7 +408,9 @@ mod tests { Query::Term { field: Some(f), .. } | Query::Wildcard { field: Some(f), .. } | Query::Fuzzy { field: Some(f), .. } => f == field, - Query::Boolean { clauses } => clauses.iter().any(|(_, c)| query_mentions_field(c, field)), + Query::Boolean { clauses } => { + clauses.iter().any(|(_, c)| query_mentions_field(c, field)) + } _ => false, } } @@ -323,36 +418,81 @@ mod tests { #[test] fn must_requires_all_clauses() { // vpn AND lang=es => only doc0 - let q = Query::Boolean { clauses: vec![ - (Occur::Must, Query::Term { field: Some("title".into()), text: "vpn".into() }), - (Occur::Must, Query::Term { field: Some("lang_key".into()), text: "es".into() }), - ]}; + let q = Query::Boolean { + clauses: vec![ + ( + Occur::Must, + Query::Term { + field: Some("title".into()), + text: "vpn".into(), + }, + ), + ( + Occur::Must, + Query::Term { + field: Some("lang_key".into()), + text: "es".into(), + }, + ), + ], + }; assert_eq!(ids(&search(&corpus(), &q, 0.0, 100)), vec![0]); } #[test] fn should_unions_when_no_must() { - let q = Query::Boolean { clauses: vec![ - (Occur::Should, Query::Term { field: Some("title".into()), text: "vpn".into() }), - (Occur::Should, Query::Term { field: Some("title".into()), text: "mysql".into() }), - ]}; + let q = Query::Boolean { + clauses: vec![ + ( + Occur::Should, + Query::Term { + field: Some("title".into()), + text: "vpn".into(), + }, + ), + ( + Occur::Should, + Query::Term { + field: Some("title".into()), + text: "mysql".into(), + }, + ), + ], + }; assert_eq!(ids(&search(&corpus(), &q, 0.0, 100)), vec![0, 1, 2]); } #[test] fn mustnot_excludes() { // vpn AND NOT lang=en => doc0 (doc1 excluded) - let q = Query::Boolean { clauses: vec![ - (Occur::Must, Query::Term { field: Some("title".into()), text: "vpn".into() }), - (Occur::MustNot, Query::Term { field: Some("lang_key".into()), text: "en".into() }), - ]}; + let q = Query::Boolean { + clauses: vec![ + ( + Occur::Must, + Query::Term { + field: Some("title".into()), + text: "vpn".into(), + }, + ), + ( + Occur::MustNot, + Query::Term { + field: Some("lang_key".into()), + text: "en".into(), + }, + ), + ], + }; assert_eq!(ids(&search(&corpus(), &q, 0.0, 100)), vec![0]); } #[test] fn all_fields_term_searches_every_indexed_field() { // field None => searches "es" in all indexed fields; matches lang_key of doc0 and doc2 - let q = Query::Term { field: None, text: "es".into() }; + let q = Query::Term { + field: None, + text: "es".into(), + }; assert_eq!(ids(&search(&corpus(), &q, 0.0, 100)), vec![0, 2]); } @@ -379,7 +519,11 @@ mod tests { // a where group with occur=Should is OPTIONAL: it boosts but does NOT filter. // build_query suffixes the raw field "lang" -> term over "lang_key". let mut p = params("vpn"); - p.where_groups = vec![WhereGroup { field: "lang".into(), values: vec!["es".into()], occur: Occur::Should }]; + p.where_groups = vec![WhereGroup { + field: "lang".into(), + values: vec!["es".into()], + occur: Occur::Should, + }]; let q = build_query(&p).unwrap(); assert_eq!(ids(&search(&corpus(), &q, 0.0, 100)), vec![0, 1]); } @@ -388,7 +532,11 @@ mod tests { fn build_query_where_must_narrows() { // occur=Must does filter (intersection): vpn AND lang=es => {0}. let mut p = params("vpn"); - p.where_groups = vec![WhereGroup { field: "lang".into(), values: vec!["es".into()], occur: Occur::Must }]; + p.where_groups = vec![WhereGroup { + field: "lang".into(), + values: vec!["es".into()], + occur: Occur::Must, + }]; let q = build_query(&p).unwrap(); assert_eq!(ids(&search(&corpus(), &q, 0.0, 100)), vec![0]); } @@ -396,7 +544,11 @@ mod tests { #[test] fn build_query_where_mustnot() { let mut p = params("vpn"); - p.where_groups = vec![WhereGroup { field: "lang".into(), values: vec!["en".into()], occur: Occur::MustNot }]; + p.where_groups = vec![WhereGroup { + field: "lang".into(), + values: vec!["en".into()], + occur: Occur::MustNot, + }]; let q = build_query(&p).unwrap(); assert_eq!(ids(&search(&corpus(), &q, 0.0, 100)), vec![0]); } @@ -409,7 +561,11 @@ mod tests { #[test] fn build_query_empty_field_is_error() { let mut p = params("vpn"); - p.where_groups = vec![WhereGroup { field: "".into(), values: vec!["x".into()], occur: Occur::Should }]; + p.where_groups = vec![WhereGroup { + field: "".into(), + values: vec!["x".into()], + occur: Occur::Should, + }]; assert!(matches!(build_query(&p), Err(QueryError::EmptyField))); } @@ -417,10 +573,17 @@ mod tests { fn build_query_where_suffixes_key_unconditionally() { // WHERE always appends "_key" (parity with the host's WHERE-clause builder): "status" -> "status_key". let mut p = params("x"); - p.where_groups = vec![WhereGroup { field: "status".into(), values: vec!["1".into()], occur: Occur::Must }]; + p.where_groups = vec![WhereGroup { + field: "status".into(), + values: vec!["1".into()], + occur: Occur::Must, + }]; let q = build_query(&p).unwrap(); // the tree must contain a Term over "status_key" - assert!(query_mentions_field(&q, "status_key"), "WHERE must suffix _key"); + assert!( + query_mentions_field(&q, "status_key"), + "WHERE must suffix _key" + ); } #[test] @@ -428,13 +591,28 @@ mod tests { // IN uses key-field naming: "cat" -> "cat_key"; "id_key" stays "id_key" (already contains it). let mut p = params("x"); p.in_groups = vec![ - InGroup { field: "cat".into(), values: vec!["1".into()] }, - InGroup { field: "id_key".into(), values: vec!["2".into()] }, + InGroup { + field: "cat".into(), + values: vec!["1".into()], + }, + InGroup { + field: "id_key".into(), + values: vec!["2".into()], + }, ]; let q = build_query(&p).unwrap(); - assert!(query_mentions_field(&q, "cat_key"), "IN must suffix _key when missing"); - assert!(query_mentions_field(&q, "id_key"), "IN must not duplicate _key"); - assert!(!query_mentions_field(&q, "id_key_key"), "IN must not duplicate _key"); + assert!( + query_mentions_field(&q, "cat_key"), + "IN must suffix _key when missing" + ); + assert!( + query_mentions_field(&q, "id_key"), + "IN must not duplicate _key" + ); + assert!( + !query_mentions_field(&q, "id_key_key"), + "IN must not duplicate _key" + ); } /// corpus to test score normalization: doc0 with high tf and a short field scores @@ -453,18 +631,32 @@ mod tests { fn search_normalizes_top_hit_to_one() { // scale parity with ZSL (Lucene.php:982-986): the top hit's score is brought to 1.0 // and the rest to (0,1). It is monotonic: it does NOT change the order. - let q = Query::Term { field: Some("title".into()), text: "vpn".into() }; + let q = Query::Term { + field: Some("title".into()), + text: "vpn".into(), + }; let hits = search(&score_corpus(), &q, 0.0, 100); assert_eq!(hits.len(), 2); - assert!((hits[0].score - 1.0).abs() < 1e-6, "top a 1.0, got {}", hits[0].score); - assert!(hits[1].score > 0.0 && hits[1].score < 1.0, "resto en (0,1), got {}", hits[1].score); + assert!( + (hits[0].score - 1.0).abs() < 1e-6, + "top a 1.0, got {}", + hits[0].score + ); + assert!( + hits[1].score > 0.0 && hits[1].score < 1.0, + "resto en (0,1), got {}", + hits[1].score + ); } #[test] fn search_min_score_filters_on_normalized_scale() { // raw scores are small (~0.1); on the normalized [0,1] scale a min_score calibrated // to ZSL behaves the same. Without normalization, min_score=0.5 would empty everything. - let q = Query::Term { field: Some("title".into()), text: "vpn".into() }; + let q = Query::Term { + field: Some("title".into()), + text: "vpn".into(), + }; assert!( !search(&score_corpus(), &q, 0.5, 100).is_empty(), "min_score=0.5 must not empty out on the normalized scale" diff --git a/sdsearch-core/src/score.rs b/sdsearch-core/src/score.rs index 9d5769d..ff32993 100644 --- a/sdsearch-core/src/score.rs +++ b/sdsearch-core/src/score.rs @@ -31,7 +31,10 @@ pub fn score_term( doc_id: usize, term_freq: u32, ) -> f32 { - let idf = idf(index.total_docs() as f32, index.doc_freq(field, term) as f32); + let idf = idf( + index.total_docs() as f32, + index.doc_freq(field, term) as f32, + ); score_with_idf(idf, term_freq, index.field_len(doc_id, field)) } diff --git a/sdsearch-core/src/search.rs b/sdsearch-core/src/search.rs index 7f7cd44..59f265e 100644 --- a/sdsearch-core/src/search.rs +++ b/sdsearch-core/src/search.rs @@ -21,11 +21,19 @@ pub struct Hit { /// raw scores (doc_id, score) of a term in a field. No sort/filter/hydration. /// The idf is computed ONCE (constant over the posting list), not per doc. pub(crate) fn term_scores(index: &impl IndexReader, field: &str, term: &str) -> Vec<(usize, f32)> { - let idf = idf(index.total_docs() as f32, index.doc_freq(field, term) as f32); + let idf = idf( + index.total_docs() as f32, + index.doc_freq(field, term) as f32, + ); index .postings_for(field, term) .into_iter() - .map(|(doc_id, tf)| (doc_id, score_with_idf(idf, tf, index.field_len(doc_id, field)))) + .map(|(doc_id, tf)| { + ( + doc_id, + score_with_idf(idf, tf, index.field_len(doc_id, field)), + ) + }) .collect() } @@ -38,7 +46,10 @@ pub(crate) fn union_scores( ) -> HashMap { let mut scored: HashMap = HashMap::new(); for term in terms { - let idf = idf(index.total_docs() as f32, index.doc_freq(field, term) as f32); + let idf = idf( + index.total_docs() as f32, + index.doc_freq(field, term) as f32, + ); for (doc_id, tf) in index.postings_for(field, term) { *scored.entry(doc_id).or_insert(0.0) += score_with_idf(idf, tf, index.field_len(doc_id, field)); @@ -131,7 +142,9 @@ pub(crate) fn fuzzy_terms( const MAX_FUZZY_TERMS: usize = 1024; if matched.len() > MAX_FUZZY_TERMS { matched.sort_by(|a, b| { - b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal).then(a.1.cmp(&b.1)) + b.0.partial_cmp(&a.0) + .unwrap_or(std::cmp::Ordering::Equal) + .then(a.1.cmp(&b.1)) }); matched.truncate(MAX_FUZZY_TERMS); } @@ -170,8 +183,13 @@ pub(crate) fn phrase_scores( for doc in candidates { let first = per_term[0].0.get(&doc).unwrap_or(&empty); let is_match = first.iter().any(|&p| { - (1..terms.len()) - .all(|i| per_term[i].0.get(&doc).unwrap_or(&empty).contains(&(p + i as u32))) + (1..terms.len()).all(|i| { + per_term[i] + .0 + .get(&doc) + .unwrap_or(&empty) + .contains(&(p + i as u32)) + }) }); if is_match { let s: f32 = (0..terms.len()) @@ -194,8 +212,10 @@ pub(crate) fn finalize( min_score: f32, limit: usize, ) -> Vec { - let mut ranked: Vec<(usize, f32)> = - scored.into_iter().filter(|(_, s)| *s >= min_score).collect(); + let mut ranked: Vec<(usize, f32)> = scored + .into_iter() + .filter(|(_, s)| *s >= min_score) + .collect(); ranked.sort_by(|a, b| { b.1.partial_cmp(&a.1) .unwrap_or(std::cmp::Ordering::Equal) @@ -204,7 +224,11 @@ pub(crate) fn finalize( ranked.truncate(limit); ranked .into_iter() - .map(|(id, score)| Hit { id, score, fields: index.stored_fields(id) }) + .map(|(id, score)| Hit { + id, + score, + fields: index.stored_fields(id), + }) .collect() } @@ -311,7 +335,10 @@ mod tests { #[test] fn returns_stored_fields() { let hits = term_query(&build(), "body", "foo", 0.0, 10); - assert_eq!(hits[0].fields.get("body").map(String::as_str), Some("foo foo baz")); + assert_eq!( + hits[0].fields.get("body").map(String::as_str), + Some("foo foo baz") + ); } #[test] @@ -331,7 +358,12 @@ mod tests { fn wildcard_corpus() -> MemoryIndex { let mut idx = MemoryIndex::new(); - for text in ["testing guide", "tested feature", "text editor", "team meeting"] { + for text in [ + "testing guide", + "tested feature", + "text editor", + "team meeting", + ] { let mut d = Document::new(); d.add("body", text, FieldKind::Text); idx.add_document(d); diff --git a/sdsearch-core/src/segment.rs b/sdsearch-core/src/segment.rs index c41c0bd..7138255 100644 --- a/sdsearch-core/src/segment.rs +++ b/sdsearch-core/src/segment.rs @@ -39,9 +39,18 @@ impl Segment { for field in &meta.fields { let bytes = std::fs::read(dir.join(format!("terms.{field}.fst")))?; fsts.insert(field.clone(), fst::Map::new(bytes).map_err(io)?); - postings.insert(field.clone(), std::fs::read(dir.join(format!("postings.{field}.bin")))?); + postings.insert( + field.clone(), + std::fs::read(dir.join(format!("postings.{field}.bin")))?, + ); } - Ok(Segment { num_docs: meta.num_docs, fsts, postings, lengths, stored }) + Ok(Segment { + num_docs: meta.num_docs, + fsts, + postings, + lengths, + stored, + }) } fn offset_of(&self, field: &str, term: &str) -> Option { @@ -57,7 +66,8 @@ impl Segment { let data = &self.postings[field]; let mut pos = off as usize; let doc_freq = read_vint(data, &mut pos)? as usize; - let mut out = Vec::with_capacity(checked_capacity(doc_freq, data.len().saturating_sub(pos))); + let mut out = + Vec::with_capacity(checked_capacity(doc_freq, data.len().saturating_sub(pos))); let mut prev = 0usize; for _ in 0..doc_freq { let delta = read_vint(data, &mut pos)? as usize; @@ -73,7 +83,12 @@ impl Segment { } /// Fallible core of `positions_for` (see `try_postings_for`). - fn try_positions_for(&self, field: &str, term: &str, doc_id: usize) -> std::io::Result> { + fn try_positions_for( + &self, + field: &str, + term: &str, + doc_id: usize, + ) -> std::io::Result> { let Some(off) = self.offset_of(field, term) else { return Ok(Vec::new()); }; @@ -85,7 +100,8 @@ impl Segment { let delta = read_vint(data, &mut pos)? as usize; let freq = read_vint(data, &mut pos)? as usize; let d = prev + delta; - let mut positions = Vec::with_capacity(checked_capacity(freq, data.len().saturating_sub(pos))); + let mut positions = + Vec::with_capacity(checked_capacity(freq, data.len().saturating_sub(pos))); for _ in 0..freq { positions.push(read_vint(data, &mut pos)? as u32); } @@ -121,7 +137,11 @@ impl IndexReader for Segment { } fn field_len(&self, doc_id: usize, field: &str) -> u32 { - self.lengths.get(field).and_then(|v| v.get(doc_id)).copied().unwrap_or(0) + self.lengths + .get(field) + .and_then(|v| v.get(doc_id)) + .copied() + .unwrap_or(0) } fn stored_fields(&self, doc_id: usize) -> HashMap { @@ -149,7 +169,8 @@ impl IndexReader for Segment { /// positions of `term` in `field` for `doc_id`, ascending order. fn positions_for(&self, field: &str, term: &str, doc_id: usize) -> Vec { // degrade: corrupt postings yield no positions rather than a panic across FFI - self.try_positions_for(field, term, doc_id).unwrap_or_default() + self.try_positions_for(field, term, doc_id) + .unwrap_or_default() } fn indexed_fields(&self) -> Vec { @@ -240,7 +261,10 @@ mod tests { assert_eq!(seg.positions_for("body", "quick", 0), vec![0, 3]); assert_eq!(seg.positions_for("body", "brown", 0), vec![1]); - assert_eq!(seg.positions_for("body", "quick", 0), mem.positions_for("body", "quick", 0)); + assert_eq!( + seg.positions_for("body", "quick", 0), + mem.positions_for("body", "quick", 0) + ); // freq is still correct after skipping positions assert_eq!(seg.postings_for("body", "quick"), vec![(0, 2)]); std::fs::remove_dir_all(&dir).unwrap(); diff --git a/sdsearch-core/src/serialize.rs b/sdsearch-core/src/serialize.rs index 37379cd..4270b68 100644 --- a/sdsearch-core/src/serialize.rs +++ b/sdsearch-core/src/serialize.rs @@ -52,7 +52,17 @@ mod tests { #[test] fn roundtrip_boundary_values() { - for v in [0u64, 1, 127, 128, 300, 16_383, 16_384, 1_000_000, u32::MAX as u64] { + for v in [ + 0u64, + 1, + 127, + 128, + 300, + 16_383, + 16_384, + 1_000_000, + u32::MAX as u64, + ] { let mut buf = Vec::new(); write_vint(&mut buf, v); let mut pos = 0; diff --git a/sdsearch-core/src/zsl/bytes.rs b/sdsearch-core/src/zsl/bytes.rs index 0336c4e..fc67622 100644 --- a/sdsearch-core/src/zsl/bytes.rs +++ b/sdsearch-core/src/zsl/bytes.rs @@ -33,7 +33,10 @@ use std::io::{self, ErrorKind}; /// Error for a read that runs off the end of the buffer. #[inline] pub fn truncated(pos: usize) -> io::Error { - io::Error::new(ErrorKind::UnexpectedEof, format!("truncated index data at offset {pos}")) + io::Error::new( + ErrorKind::UnexpectedEof, + format!("truncated index data at offset {pos}"), + ) } /// Clamp a file-derived element count to what could physically fit in the @@ -80,7 +83,10 @@ pub fn read_byte(data: &[u8], pos: &mut usize) -> io::Result { /// modified-UTF-8 = UTF-8 except NUL is encoded as C0 80. pub fn read_modified_utf8(data: &[u8], pos: &mut usize) -> io::Result { let char_count = read_vint(data, pos)? as usize; - let mut s = String::with_capacity(checked_capacity(char_count, data.len().saturating_sub(*pos))); + let mut s = String::with_capacity(checked_capacity( + char_count, + data.len().saturating_sub(*pos), + )); for _ in 0..char_count { let b0 = read_byte(data, pos)?; if b0 & 0x80 == 0 { @@ -93,7 +99,8 @@ pub fn read_modified_utf8(data: &[u8], pos: &mut usize) -> io::Result { // 3-byte (BMP). let b1 = read_byte(data, pos)?; let b2 = read_byte(data, pos)?; - let cp = (((b0 & 0x0F) as u32) << 12) | (((b1 & 0x3F) as u32) << 6) | ((b2 & 0x3F) as u32); + let cp = + (((b0 & 0x0F) as u32) << 12) | (((b1 & 0x3F) as u32) << 6) | ((b2 & 0x3F) as u32); s.push(char::from_u32(cp).unwrap_or('\u{FFFD}')); } else { // 4-byte (supplementary plane). ZSL's PHP writer stores standard UTF-8 (not Java's @@ -157,14 +164,27 @@ mod tests { // Java surrogates) → the reader must decode them as 1 code point (real documents with // emoji drifted without the 4-byte branch). for s in [ - "", "hi", "über", "a€b", "na\u{0}me", "TICKET-12345", "user@example.com", - "a\u{1F600}b", "\u{20000}", "mix \u{1F4A9} end", "über\u{1F680}", + "", + "hi", + "über", + "a€b", + "na\u{0}me", + "TICKET-12345", + "user@example.com", + "a\u{1F600}b", + "\u{20000}", + "mix \u{1F4A9} end", + "über\u{1F680}", ] { let mut buf = Vec::new(); write_modified_utf8(&mut buf, s); // the charCount prefix must be the number of code points let mut pos = 0; - assert_eq!(read_modified_utf8(&buf, &mut pos).unwrap(), s, "roundtrip {s:?}"); + assert_eq!( + read_modified_utf8(&buf, &mut pos).unwrap(), + s, + "roundtrip {s:?}" + ); assert_eq!(pos, buf.len(), "consumed all bytes of {s:?}"); } } @@ -172,10 +192,16 @@ mod tests { #[test] fn reads_big_endian_integers() { let mut pos = 0; - assert_eq!(read_u32_be(&[0x00, 0x00, 0x01, 0x00], &mut pos).unwrap(), 256); + assert_eq!( + read_u32_be(&[0x00, 0x00, 0x01, 0x00], &mut pos).unwrap(), + 256 + ); assert_eq!(pos, 4); let mut pos = 0; - assert_eq!(read_i32_be(&[0xFF, 0xFF, 0xFF, 0xFD], &mut pos).unwrap(), -3); + assert_eq!( + read_i32_be(&[0xFF, 0xFF, 0xFF, 0xFD], &mut pos).unwrap(), + -3 + ); let mut pos = 0; assert_eq!(read_u64_be(&[0, 0, 0, 0, 0, 0, 0, 5], &mut pos).unwrap(), 5); } diff --git a/sdsearch-core/src/zsl/cfs.rs b/sdsearch-core/src/zsl/cfs.rs index 702869d..5ba3c89 100644 --- a/sdsearch-core/src/zsl/cfs.rs +++ b/sdsearch-core/src/zsl/cfs.rs @@ -28,11 +28,17 @@ impl CompoundFile { let mut entries = Vec::with_capacity(checked_capacity(count, total)); for i in 0..count { let start = raw[i].0 as usize; - let end = if i + 1 < count { raw[i + 1].0 as usize } else { total }; + let end = if i + 1 < count { + raw[i + 1].0 as usize + } else { + total + }; if start > end || end > total { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!("cfs sub-file offset out of range: start={start} end={end} total={total}"), + format!( + "cfs sub-file offset out of range: start={start} end={end} total={total}" + ), )); } entries.push((raw[i].1.clone(), start, end)); @@ -58,7 +64,10 @@ mod tests { use std::path::PathBuf; fn fixture_cfs() -> PathBuf { - let dir = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index")); + let dir = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index" + )); std::fs::read_dir(&dir) .unwrap() .filter_map(|e| e.ok().map(|e| e.path())) diff --git a/sdsearch-core/src/zsl/deletes.rs b/sdsearch-core/src/zsl/deletes.rs index 916f42a..38babb4 100644 --- a/sdsearch-core/src/zsl/deletes.rs +++ b/sdsearch-core/src/zsl/deletes.rs @@ -31,7 +31,9 @@ impl DeletedDocs { )); } let _bit_count = read_i32_be(del, &mut pos)?; - Ok(DeletedDocs { bits: del.get(pos..).unwrap_or(&[]).to_vec() }) + Ok(DeletedDocs { + bits: del.get(pos..).unwrap_or(&[]).to_vec(), + }) } pub fn none() -> DeletedDocs { diff --git a/sdsearch-core/src/zsl/fields.rs b/sdsearch-core/src/zsl/fields.rs index 167db90..26c1245 100644 --- a/sdsearch-core/src/zsl/fields.rs +++ b/sdsearch-core/src/zsl/fields.rs @@ -14,7 +14,10 @@ pub fn read_field_infos(fnm: &[u8]) -> std::io::Result> { for _ in 0..count { let name = read_modified_utf8(fnm, &mut pos)?; let flags = read_byte(fnm, &mut pos)?; - out.push(FieldInfo { name, is_indexed: flags & 0x01 != 0 }); + out.push(FieldInfo { + name, + is_indexed: flags & 0x01 != 0, + }); } Ok(out) } @@ -37,8 +40,14 @@ mod tests { assert_eq!( fields, vec![ - FieldInfo { name: "title".into(), is_indexed: true }, - FieldInfo { name: "id_attr".into(), is_indexed: false }, + FieldInfo { + name: "title".into(), + is_indexed: true + }, + FieldInfo { + name: "id_attr".into(), + is_indexed: false + }, ] ); } diff --git a/sdsearch-core/src/zsl/index.rs b/sdsearch-core/src/zsl/index.rs index 4e2adbd..f168e16 100644 --- a/sdsearch-core/src/zsl/index.rs +++ b/sdsearch-core/src/zsl/index.rs @@ -7,8 +7,8 @@ use std::collections::HashMap; use std::path::Path; struct Entry { - base: usize, // global doc id of the segment's first doc - max_doc: usize, // size of the id space (incl. deletes) + base: usize, // global doc id of the segment's first doc + max_doc: usize, // size of the id space (incl. deletes) seg: ZslSegment, } @@ -33,7 +33,11 @@ impl ZslIndex { } let num_docs = entries.iter().map(|e| e.seg.num_docs()).sum(); let total_docs = entries.iter().map(|e| e.max_doc).sum(); - Ok(ZslIndex { entries, num_docs, total_docs }) + Ok(ZslIndex { + entries, + num_docs, + total_docs, + }) } /// (entry, local id) that owns the global id, or None if out of range. @@ -55,7 +59,10 @@ impl IndexReader for ZslIndex { } fn doc_freq(&self, field: &str, term: &str) -> usize { - self.entries.iter().map(|e| e.seg.doc_freq(field, term)).sum() + self.entries + .iter() + .map(|e| e.seg.doc_freq(field, term)) + .sum() } fn postings_for(&self, field: &str, term: &str) -> Vec<(usize, u32)> { @@ -124,7 +131,10 @@ mod tests { use std::path::PathBuf; fn kb() -> PathBuf { - PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_kb")) + PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_kb" + )) } #[test] @@ -133,7 +143,10 @@ mod tests { let seg = ZslSegment::open(&kb()).unwrap(); assert_eq!(idx.num_docs(), seg.num_docs()); assert_eq!(idx.doc_freq("title", "vpn"), seg.doc_freq("title", "vpn")); - assert_eq!(idx.postings_for("title", "vpn"), seg.postings_for("title", "vpn")); + assert_eq!( + idx.postings_for("title", "vpn"), + seg.postings_for("title", "vpn") + ); // stored routing: the same doc returns the same id_key let d0 = idx.postings_for("title", "vpn")[0].0; assert_eq!(idx.stored_fields(d0), seg.stored_fields(d0)); diff --git a/sdsearch-core/src/zsl/mod.rs b/sdsearch-core/src/zsl/mod.rs index 3114e77..828d7cf 100644 --- a/sdsearch-core/src/zsl/mod.rs +++ b/sdsearch-core/src/zsl/mod.rs @@ -7,8 +7,8 @@ pub mod index; pub mod norms; pub mod postings; pub mod runner; -pub mod segments; pub mod segment; +pub mod segments; pub mod stored; pub mod terms; pub mod writer; diff --git a/sdsearch-core/src/zsl/norms.rs b/sdsearch-core/src/zsl/norms.rs index dd0dbb6..3b68735 100644 --- a/sdsearch-core/src/zsl/norms.rs +++ b/sdsearch-core/src/zsl/norms.rs @@ -1,7 +1,11 @@ //! Norms reader (.nrm): one norm byte per doc per indexed field. use std::collections::HashMap; -pub fn read_norms(nrm: &[u8], indexed_fields: &[String], num_docs: usize) -> HashMap> { +pub fn read_norms( + nrm: &[u8], + indexed_fields: &[String], + num_docs: usize, +) -> HashMap> { let mut out = HashMap::new(); // header: 'NRM' + 1 format byte let mut pos = 4usize; diff --git a/sdsearch-core/src/zsl/postings.rs b/sdsearch-core/src/zsl/postings.rs index 5c39f42..91572cc 100644 --- a/sdsearch-core/src/zsl/postings.rs +++ b/sdsearch-core/src/zsl/postings.rs @@ -7,12 +7,19 @@ use crate::zsl::terms::TermInfo; /// otherwise a second `VInt(freq)` is read. pub fn read_freqs(frq: &[u8], info: &TermInfo) -> std::io::Result> { let mut pos = info.freq_pointer as usize; - let mut out = Vec::with_capacity(checked_capacity(info.doc_freq as usize, frq.len().saturating_sub(pos))); + let mut out = Vec::with_capacity(checked_capacity( + info.doc_freq as usize, + frq.len().saturating_sub(pos), + )); let mut prev = 0usize; for _ in 0..info.doc_freq { let v = read_vint(frq, &mut pos)?; let doc_delta = (v >> 1) as usize; - let freq = if v & 1 == 1 { 1 } else { read_vint(frq, &mut pos)? as u32 }; + let freq = if v & 1 == 1 { + 1 + } else { + read_vint(frq, &mut pos)? as u32 + }; let doc_id = prev + doc_delta; out.push((doc_id, freq)); prev = doc_id; @@ -22,16 +29,28 @@ pub fn read_freqs(frq: &[u8], info: &TermInfo) -> std::io::Result std::io::Result> { +pub fn read_positions( + frq: &[u8], + prx: &[u8], + info: &TermInfo, + doc_id: usize, +) -> std::io::Result> { let mut fpos = info.freq_pointer as usize; let mut ppos = info.prox_pointer as usize; let mut prev = 0usize; for _ in 0..info.doc_freq { let v = read_vint(frq, &mut fpos)?; let doc_delta = (v >> 1) as usize; - let freq = if v & 1 == 1 { 1 } else { read_vint(frq, &mut fpos)? as u32 }; + let freq = if v & 1 == 1 { + 1 + } else { + read_vint(frq, &mut fpos)? as u32 + }; let d = prev + doc_delta; - let mut positions = Vec::with_capacity(checked_capacity(freq as usize, prx.len().saturating_sub(ppos))); + let mut positions = Vec::with_capacity(checked_capacity( + freq as usize, + prx.len().saturating_sub(ppos), + )); let mut prev_pos = 0u32; for _ in 0..freq { prev_pos += read_vint(prx, &mut ppos)? as u32; @@ -47,17 +66,31 @@ pub fn read_positions(frq: &[u8], prx: &[u8], info: &TermInfo, doc_id: usize) -> /// Decodes ALL positions of a term in a single pass: doc -> positions. /// Avoids re-walking `.frq`/`.prx` from the pointer for each doc (which made phrase O(C·docFreq)). -pub fn read_all_positions(frq: &[u8], prx: &[u8], info: &TermInfo) -> std::io::Result)>> { +pub fn read_all_positions( + frq: &[u8], + prx: &[u8], + info: &TermInfo, +) -> std::io::Result)>> { let mut fpos = info.freq_pointer as usize; let mut ppos = info.prox_pointer as usize; let mut prev = 0usize; - let mut out = Vec::with_capacity(checked_capacity(info.doc_freq as usize, frq.len().saturating_sub(fpos))); + let mut out = Vec::with_capacity(checked_capacity( + info.doc_freq as usize, + frq.len().saturating_sub(fpos), + )); for _ in 0..info.doc_freq { let v = read_vint(frq, &mut fpos)?; let doc_delta = (v >> 1) as usize; - let freq = if v & 1 == 1 { 1 } else { read_vint(frq, &mut fpos)? as u32 }; + let freq = if v & 1 == 1 { + 1 + } else { + read_vint(frq, &mut fpos)? as u32 + }; let d = prev + doc_delta; - let mut positions = Vec::with_capacity(checked_capacity(freq as usize, prx.len().saturating_sub(ppos))); + let mut positions = Vec::with_capacity(checked_capacity( + freq as usize, + prx.len().saturating_sub(ppos), + )); let mut prev_pos = 0u32; for _ in 0..freq { prev_pos += read_vint(prx, &mut ppos)? as u32; @@ -78,18 +111,31 @@ mod tests { use std::path::PathBuf; fn cfs() -> CompoundFile { - let dir = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index")); - let path = std::fs::read_dir(&dir).unwrap() + let dir = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index" + )); + let path = std::fs::read_dir(&dir) + .unwrap() .filter_map(|e| e.ok().map(|e| e.path())) - .find(|p| p.extension().map(|x| x == "cfs").unwrap_or(false)).unwrap(); + .find(|p| p.extension().map(|x| x == "cfs").unwrap_or(false)) + .unwrap(); CompoundFile::open(&path).unwrap() } #[test] fn freqs_match_oracle_for_new() { let cf = cfs(); - let sub = |ext: &str| cf.sub(&cf.names().into_iter().find(|n| n.ends_with(ext)).unwrap()).unwrap().to_vec(); - let names: Vec = read_field_infos(&sub(".fnm")).unwrap().into_iter().map(|f| f.name).collect(); + let sub = |ext: &str| { + cf.sub(&cf.names().into_iter().find(|n| n.ends_with(ext)).unwrap()) + .unwrap() + .to_vec() + }; + let names: Vec = read_field_infos(&sub(".fnm")) + .unwrap() + .into_iter() + .map(|f| f.name) + .collect(); let dict = TermDict::read(&sub(".tis"), &names).unwrap(); let info = dict.info("title", "new").unwrap(); let freqs = read_freqs(&sub(".frq"), info).unwrap(); @@ -99,7 +145,11 @@ mod tests { #[test] fn read_freqs_errors_on_truncated_frq() { - let info = TermInfo { doc_freq: 3, freq_pointer: 0, prox_pointer: 0 }; + let info = TermInfo { + doc_freq: 3, + freq_pointer: 0, + prox_pointer: 0, + }; assert!(read_freqs(&[], &info).is_err()); } } diff --git a/sdsearch-core/src/zsl/runner.rs b/sdsearch-core/src/zsl/runner.rs index 8624e92..a281dd5 100644 --- a/sdsearch-core/src/zsl/runner.rs +++ b/sdsearch-core/src/zsl/runner.rs @@ -37,7 +37,13 @@ fn fallback_query(text: &str) -> Option { let mut clauses: Vec<(Occur, Query)> = Vec::new(); for tok in analyze(text) { if seen.insert(tok.clone()) { - clauses.push((Occur::Should, Query::Term { field: None, text: tok })); + clauses.push(( + Occur::Should, + Query::Term { + field: None, + text: tok, + }, + )); } } if clauses.is_empty() { @@ -54,12 +60,19 @@ mod tests { use std::path::PathBuf; fn multiseg() -> PathBuf { - PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_multiseg")) + PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_multiseg" + )) } fn params(text: &str) -> QueryParams { QueryParams { - text: text.into(), where_groups: vec![], in_groups: vec![], - fuzzy_similarity: 0.5, fuzzy_prefix_len: 3, wildcard_min_prefix: 0, + text: text.into(), + where_groups: vec![], + in_groups: vec![], + fuzzy_similarity: 0.5, + fuzzy_prefix_len: 3, + wildcard_min_prefix: 0, } } fn ids(hits: &[Hit]) -> Vec { @@ -80,7 +93,10 @@ mod tests { // text "vpn" (Must) + in cat=999 (Must, no doc) => empty primary; // the all-fields fallback "vpn" recovers [0,2]. let mut p = params("vpn"); - p.in_groups = vec![InGroup { field: "cat".into(), values: vec!["999".into()] }]; + p.in_groups = vec![InGroup { + field: "cat".into(), + values: vec!["999".into()], + }]; let hits = search_index(&multiseg(), &p, 0.0, 0).unwrap(); assert_eq!(ids(&hits), vec![0, 2]); } diff --git a/sdsearch-core/src/zsl/segment.rs b/sdsearch-core/src/zsl/segment.rs index c7c1d63..81be422 100644 --- a/sdsearch-core/src/zsl/segment.rs +++ b/sdsearch-core/src/zsl/segment.rs @@ -80,7 +80,11 @@ impl ZslSegment { } /// opens a named segment (`.cfs`) and its `.del` per `del_gen`. - pub fn open_named(index_dir: &Path, seg_name: &str, del_gen: i64) -> std::io::Result { + pub fn open_named( + index_dir: &Path, + seg_name: &str, + del_gen: i64, + ) -> std::io::Result { let cfs_path = index_dir.join(format!("{seg_name}.cfs")); let del_path = match del_gen { -1 => None, @@ -110,8 +114,11 @@ impl ZslSegment { let fdx_name = name_ending(".fdx").ok_or_else(|| std::io::Error::other("no .fdx"))?; let num_docs_total = cfs.sub(&fdx_name).unwrap().len() / 8; - let indexed: Vec = - fields.iter().filter(|f| f.is_indexed).map(|f| f.name.clone()).collect(); + let indexed: Vec = fields + .iter() + .filter(|f| f.is_indexed) + .map(|f| f.name.clone()) + .collect(); let norms = match name_ending(".nrm") { Some(n) => read_norms(cfs.sub(&n).unwrap(), &indexed, num_docs_total), None => HashMap::new(), @@ -119,7 +126,10 @@ impl ZslSegment { // .del lives OUTSIDE the .cfs; we load it only if the file exists. A corrupt or // unsupported (sparse) .del surfaces as an error at open time rather than a crash. - let deletes = match del_path.filter(|p| p.exists()).and_then(|p| std::fs::read(p).ok()) { + let deletes = match del_path + .filter(|p| p.exists()) + .and_then(|p| std::fs::read(p).ok()) + { Some(b) => DeletedDocs::read(&b)?, None => DeletedDocs::none(), }; @@ -129,10 +139,21 @@ impl ZslSegment { let prx_name = name_ending(".prx").unwrap_or_default(); let _ = index_dir; // the .del was already resolved above - let num_docs_live = (0..num_docs_total).filter(|&d| !deletes.is_deleted(d)).count(); + let num_docs_live = (0..num_docs_total) + .filter(|&d| !deletes.is_deleted(d)) + .count(); Ok(ZslSegment { - num_docs_total, num_docs_live, fields, dict, norms, deletes, cfs, - fdx_name, fdt_name, frq_name, prx_name, + num_docs_total, + num_docs_live, + fields, + dict, + norms, + deletes, + cfs, + fdx_name, + fdt_name, + frq_name, + prx_name, }) } } @@ -148,7 +169,10 @@ impl IndexReader for ZslSegment { } fn doc_freq(&self, field: &str, term: &str) -> usize { - self.dict.info(field, term).map(|ti| ti.doc_freq as usize).unwrap_or(0) + self.dict + .info(field, term) + .map(|ti| ti.doc_freq as usize) + .unwrap_or(0) } fn postings_for(&self, field: &str, term: &str) -> Vec<(usize, u32)> { @@ -224,8 +248,12 @@ impl IndexReader for ZslSegment { } fn indexed_fields(&self) -> Vec { - let mut v: Vec = - self.fields.iter().filter(|f| f.is_indexed).map(|f| f.name.clone()).collect(); + let mut v: Vec = self + .fields + .iter() + .filter(|f| f.is_indexed) + .map(|f| f.name.clone()) + .collect(); v.sort(); v } @@ -237,7 +265,10 @@ mod tests { use std::path::PathBuf; fn seg() -> ZslSegment { - let dir = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index")); + let dir = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index" + )); ZslSegment::open(&dir).unwrap() } @@ -251,24 +282,36 @@ mod tests { let s = seg(); // incidents index: "new" in title in all 4 docs, freq 1 each assert_eq!(s.doc_freq("title", "new"), 4); - assert_eq!(s.postings_for("title", "new"), vec![(0, 1), (1, 1), (2, 1), (3, 1)]); + assert_eq!( + s.postings_for("title", "new"), + vec![(0, 1), (1, 1), (2, 1), (3, 1)] + ); } #[test] fn stored_fields_round_trip() { let s = seg(); - assert_eq!(s.stored_fields(0).get("id_key").map(String::as_str), Some("165")); + assert_eq!( + s.stored_fields(0).get("id_key").map(String::as_str), + Some("165") + ); } #[test] fn open_named_matches_scanning_open() { - let dir = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_kb")); + let dir = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_kb" + )); // KB has a single segment "_2" with no deletes (del_gen -1) let named = ZslSegment::open_named(&dir, "_2", -1).unwrap(); let scanned = ZslSegment::open(&dir).unwrap(); assert_eq!(named.max_doc(), 20); assert_eq!(named.num_docs(), scanned.num_docs()); - assert_eq!(named.postings_for("title", "vpn"), scanned.postings_for("title", "vpn")); + assert_eq!( + named.postings_for("title", "vpn"), + scanned.postings_for("title", "vpn") + ); } #[test] @@ -289,15 +332,20 @@ mod tests { #[test] fn merge_accessors_expose_fields_deletes_norms_terms_stored() { let s = seg(); // incidents fixture, 4 docs, no deletes - // field_infos: includes title (indexed) - assert!(s.field_infos().iter().any(|f| f.name == "title" && f.is_indexed)); + // field_infos: includes title (indexed) + assert!(s + .field_infos() + .iter() + .any(|f| f.name == "title" && f.is_indexed)); // is_deleted: nothing deleted in the fixture assert!(!s.is_deleted(0)); // norm_bytes: title has a 4-byte column (one per doc) assert_eq!(s.norm_bytes("title").map(|c| c.len()), Some(4)); assert!(s.norm_bytes("campo_inexistente").is_none()); // all_terms: title:new present - assert!(s.all_terms().contains(&("title".to_string(), "new".to_string()))); + assert!(s + .all_terms() + .contains(&("title".to_string(), "new".to_string()))); // stored_raw doc 0: contains id_key="165" (same value as stored_fields) let raw = s.stored_raw(0).unwrap(); let names = s.field_infos(); diff --git a/sdsearch-core/src/zsl/segments.rs b/sdsearch-core/src/zsl/segments.rs index 6405174..91d4469 100644 --- a/sdsearch-core/src/zsl/segments.rs +++ b/sdsearch-core/src/zsl/segments.rs @@ -1,6 +1,8 @@ //! Parser for ZSL's generation files (segments.gen + segments_N). -use crate::zsl::bytes::{checked_capacity, read_byte, read_modified_utf8, read_u32_be, read_u64_be}; +use crate::zsl::bytes::{ + checked_capacity, read_byte, read_modified_utf8, read_u32_be, read_u64_be, +}; use std::path::Path; /// minimal per-segment info taken from the generation file. @@ -92,7 +94,11 @@ pub fn read_segment_infos(index_dir: &Path) -> std::io::Result> } let _is_compound_byte = read_byte(&data, &mut pos)?; - out.push(SegmentInfo { name, doc_count, del_gen }); + out.push(SegmentInfo { + name, + doc_count, + del_gen, + }); } Ok(out) } @@ -103,7 +109,10 @@ mod tests { use std::path::PathBuf; fn kb_dir() -> PathBuf { - PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_kb")) + PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_kb" + )) } #[test] diff --git a/sdsearch-core/src/zsl/stored.rs b/sdsearch-core/src/zsl/stored.rs index 3862afd..158d56d 100644 --- a/sdsearch-core/src/zsl/stored.rs +++ b/sdsearch-core/src/zsl/stored.rs @@ -1,5 +1,7 @@ //! Stored fields reader (.fdt indexed by .fdx). -use crate::zsl::bytes::{checked_capacity, read_byte, read_modified_utf8, read_u64_be, read_vint, truncated}; +use crate::zsl::bytes::{ + checked_capacity, read_byte, read_modified_utf8, read_u64_be, read_vint, truncated, +}; use crate::zsl::fields::FieldInfo; use std::collections::HashMap; @@ -29,7 +31,10 @@ pub fn read_stored_raw(fdx: &[u8], fdt: &[u8], doc_id: usize) -> std::io::Result let fdt_off = read_u64_be(fdx, &mut p)? as usize; let mut pos = fdt_off; let stored_count = read_vint(fdt, &mut pos)? as usize; - out.reserve(checked_capacity(stored_count, fdt.len().saturating_sub(pos))); + out.reserve(checked_capacity( + stored_count, + fdt.len().saturating_sub(pos), + )); for _ in 0..stored_count { let field_num = read_vint(fdt, &mut pos)? as usize; let flags = read_byte(fdt, &mut pos)?; @@ -44,7 +49,12 @@ pub fn read_stored_raw(fdx: &[u8], fdt: &[u8], doc_id: usize) -> std::io::Result } else { read_modified_utf8(fdt, &mut pos)? }; - out.push(StoredRaw { field_num, value, tokenized, is_binary }); + out.push(StoredRaw { + field_num, + value, + tokenized, + is_binary, + }); } Ok(out) } @@ -75,21 +85,39 @@ mod tests { use std::path::PathBuf; fn cfs() -> CompoundFile { - let dir = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index")); - let path = std::fs::read_dir(&dir).unwrap() + let dir = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index" + )); + let path = std::fs::read_dir(&dir) + .unwrap() .filter_map(|e| e.ok().map(|e| e.path())) - .find(|p| p.extension().map(|x| x == "cfs").unwrap_or(false)).unwrap(); + .find(|p| p.extension().map(|x| x == "cfs").unwrap_or(false)) + .unwrap(); CompoundFile::open(&path).unwrap() } #[test] fn stored_fields_match_zsl_oracle_for_doc0() { let cf = cfs(); - let fnm = cf.names().into_iter().find(|n| n.ends_with(".fnm")).unwrap(); - let fdx = cf.names().into_iter().find(|n| n.ends_with(".fdx")).unwrap(); - let fdt = cf.names().into_iter().find(|n| n.ends_with(".fdt")).unwrap(); + let fnm = cf + .names() + .into_iter() + .find(|n| n.ends_with(".fnm")) + .unwrap(); + let fdx = cf + .names() + .into_iter() + .find(|n| n.ends_with(".fdx")) + .unwrap(); + let fdt = cf + .names() + .into_iter() + .find(|n| n.ends_with(".fdt")) + .unwrap(); let fields = read_field_infos(cf.sub(&fnm).unwrap()).unwrap(); - let stored = read_stored_fields(cf.sub(&fdx).unwrap(), cf.sub(&fdt).unwrap(), &fields, 0).unwrap(); + let stored = + read_stored_fields(cf.sub(&fdx).unwrap(), cf.sub(&fdt).unwrap(), &fields, 0).unwrap(); // FULL parity with what ZSL stored for doc 0 (read from the oracle). // tokenized Text fields (title, users) carry a trailing '\n' that compactText() // adds — it's a faithful part of the bytes, NOT trimmed. @@ -98,12 +126,18 @@ mod tests { fn oracle_doc0_stored() -> std::collections::HashMap { #[derive(serde::Deserialize)] - struct Oracle { docs: Vec } + struct Oracle { + docs: Vec, + } #[derive(serde::Deserialize)] - struct OracleDoc { stored: std::collections::HashMap } + struct OracleDoc { + stored: std::collections::HashMap, + } let raw = std::fs::read_to_string(concat!( - env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_expected.json" - )).expect("oracle missing"); + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_expected.json" + )) + .expect("oracle missing"); let o: Oracle = serde_json::from_str(&raw).unwrap(); o.docs.into_iter().next().unwrap().stored } diff --git a/sdsearch-core/src/zsl/terms.rs b/sdsearch-core/src/zsl/terms.rs index 6c46b72..1b97bbb 100644 --- a/sdsearch-core/src/zsl/terms.rs +++ b/sdsearch-core/src/zsl/terms.rs @@ -75,7 +75,11 @@ impl TermDict { let ft = by_field.entry(field).or_default(); ft.offsets.push(ft.text.len() as u32); ft.text.extend_from_slice(text.as_bytes()); - ft.infos.push(TermInfo { doc_freq, freq_pointer: freq_ptr, prox_pointer: prox_ptr }); + ft.infos.push(TermInfo { + doc_freq, + freq_pointer: freq_ptr, + prox_pointer: prox_ptr, + }); prev_text = text; } @@ -141,7 +145,10 @@ impl TermDict { let mut out = Vec::new(); for (field, ft) in &self.by_field { for i in 0..ft.len() { - out.push((field.clone(), String::from_utf8_lossy(ft.term(i)).into_owned())); + out.push(( + field.clone(), + String::from_utf8_lossy(ft.term(i)).into_owned(), + )); } } out @@ -156,16 +163,31 @@ mod tests { use std::path::PathBuf; fn dict() -> TermDict { - let dir = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index")); + let dir = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index" + )); let path = std::fs::read_dir(&dir) .unwrap() .filter_map(|e| e.ok().map(|e| e.path())) .find(|p| p.extension().map(|x| x == "cfs").unwrap_or(false)) .unwrap(); let cf = CompoundFile::open(&path).unwrap(); - let fnm = cf.names().into_iter().find(|n| n.ends_with(".fnm")).unwrap(); - let tis = cf.names().into_iter().find(|n| n.ends_with(".tis")).unwrap(); - let names: Vec = read_field_infos(cf.sub(&fnm).unwrap()).unwrap().into_iter().map(|f| f.name).collect(); + let fnm = cf + .names() + .into_iter() + .find(|n| n.ends_with(".fnm")) + .unwrap(); + let tis = cf + .names() + .into_iter() + .find(|n| n.ends_with(".tis")) + .unwrap(); + let names: Vec = read_field_infos(cf.sub(&fnm).unwrap()) + .unwrap() + .into_iter() + .map(|f| f.name) + .collect(); TermDict::read(cf.sub(&tis).unwrap(), &names).unwrap() } @@ -202,8 +224,14 @@ mod tests { got.sort(); // the incidents fixture has 4 docs "New workflow" + fields id_key/users/etc. // minimal stable assertion: title:new and title:workflow are present. - assert!(got.contains(&("title".to_string(), "new".to_string())), "got={got:?}"); - assert!(got.contains(&("title".to_string(), "workflow".to_string())), "got={got:?}"); + assert!( + got.contains(&("title".to_string(), "new".to_string())), + "got={got:?}" + ); + assert!( + got.contains(&("title".to_string(), "workflow".to_string())), + "got={got:?}" + ); // total count = sum of terms per field (non-empty) assert!(!got.is_empty()); } diff --git a/sdsearch-core/src/zsl/writer/cfs.rs b/sdsearch-core/src/zsl/writer/cfs.rs index 06e96f8..c312891 100644 --- a/sdsearch-core/src/zsl/writer/cfs.rs +++ b/sdsearch-core/src/zsl/writer/cfs.rs @@ -48,8 +48,7 @@ mod tests { let fnm = b"field-infos-bytes".to_vec(); let tis = b"term-dict-bytes-longer".to_vec(); let frq = vec![0u8, 1, 2, 3, 4]; - let files: Vec<(&str, &[u8])> = - vec![(".fnm", &fnm), (".tis", &tis), (".frq", &frq)]; + let files: Vec<(&str, &[u8])> = vec![(".fnm", &fnm), (".tis", &tis), (".frq", &frq)]; let cfs = write_cfs("_7", &files); diff --git a/sdsearch-core/src/zsl/writer/deletes.rs b/sdsearch-core/src/zsl/writer/deletes.rs index 096b781..dc695aa 100644 --- a/sdsearch-core/src/zsl/writer/deletes.rs +++ b/sdsearch-core/src/zsl/writer/deletes.rs @@ -17,7 +17,10 @@ pub fn write_del_file(doc_count: usize, deleted: &BTreeSet) -> Vec { } let mut out = Vec::with_capacity(8 + byte_count); write_i32_be(&mut out, doc_count as i32); - write_i32_be(&mut out, deleted.iter().filter(|&&id| id < doc_count).count() as i32); + write_i32_be( + &mut out, + deleted.iter().filter(|&&id| id < doc_count).count() as i32, + ); out.extend_from_slice(&bitmap); out } diff --git a/sdsearch-core/src/zsl/writer/fnm.rs b/sdsearch-core/src/zsl/writer/fnm.rs index b192812..4811e92 100644 --- a/sdsearch-core/src/zsl/writer/fnm.rs +++ b/sdsearch-core/src/zsl/writer/fnm.rs @@ -23,15 +23,27 @@ mod tests { #[test] fn fnm_roundtrips_names_and_indexed_flag_through_reader() { let fields = vec![ - FieldMeta { name: "title".into(), indexed: true }, - FieldMeta { name: "id_attr".into(), indexed: false }, + FieldMeta { + name: "title".into(), + indexed: true, + }, + FieldMeta { + name: "id_attr".into(), + indexed: false, + }, ]; let fnm = write_fnm(&fields); assert_eq!( read_field_infos(&fnm).unwrap(), vec![ - FieldInfo { name: "title".into(), is_indexed: true }, - FieldInfo { name: "id_attr".into(), is_indexed: false }, + FieldInfo { + name: "title".into(), + is_indexed: true + }, + FieldInfo { + name: "id_attr".into(), + is_indexed: false + }, ] ); } diff --git a/sdsearch-core/src/zsl/writer/index_writer.rs b/sdsearch-core/src/zsl/writer/index_writer.rs index 5c983b5..6bca196 100644 --- a/sdsearch-core/src/zsl/writer/index_writer.rs +++ b/sdsearch-core/src/zsl/writer/index_writer.rs @@ -20,16 +20,16 @@ use std::path::{Path, PathBuf}; pub struct IndexWriter { dir: PathBuf, - _lock: WriteLock, // released on Drop - base: Generation, // snapshot of generation N at open() + _lock: WriteLock, // released on Drop + base: Generation, // snapshot of generation N at open() base_live_docs: usize, // live docs of generation N (Σ maxDoc − deleted) /// base generation's segment table: (name, maxDoc, delGen), in segments_N order. base_segments: Vec<(String, u32, i64)>, /// buffered deletions keyed by base segment name (ids LOCAL to the segment). pending_deletes: HashMap>, - next_name_counter: u32, // starts at base.name_counter, ++ per flush + next_name_counter: u32, // starts at base.name_counter, ++ per flush flushed: Vec, // segments flushed in this session (not yet committed) - buffer: Vec, // in-RAM docs not yet flushed + buffer: Vec, // in-RAM docs not yet flushed max_buffered_docs: usize, opts: WriterOpts, } @@ -89,7 +89,10 @@ impl IndexWriter { let seg_len = *max_doc as usize; if global_doc_id < base + seg_len { let local = global_doc_id - base; - self.pending_deletes.entry(name.clone()).or_default().insert(local); + self.pending_deletes + .entry(name.clone()) + .or_default() + .insert(local); return; } base += seg_len; @@ -114,7 +117,10 @@ impl IndexWriter { } let seg_name = segments::segment_name(self.next_name_counter); let doc_count = write_segment_cfs(&self.dir, &seg_name, &self.buffer, &self.opts)?; - self.flushed.push(NewSegment { name: seg_name, doc_count: doc_count as u32 }); + self.flushed.push(NewSegment { + name: seg_name, + doc_count: doc_count as u32, + }); self.next_name_counter += 1; self.buffer.clear(); Ok(()) @@ -142,14 +148,16 @@ impl IndexWriter { // not the session doc_count (which is 0 if nothing was added). The indexing feed uses // Writer::document_count(), but the field's contract must still be correct. let total: usize = infos.iter().map(|s| s.doc_count).sum(); - return Ok(CommitReport { doc_count: total, ..commit_rep }); + return Ok(CommitReport { + doc_count: total, + ..commit_rep + }); } // 3) merge -> bytes of the merged .cfs. Name = next from name_counter. let gen = segments::read_generation(&self.dir)?; let merged_name = segments::segment_name(gen.name_counter); - let refs: Vec<(String, i64)> = - infos.iter().map(|s| (s.name.clone(), s.del_gen)).collect(); + let refs: Vec<(String, i64)> = infos.iter().map(|s| (s.name.clone(), s.del_gen)).collect(); let result = merge::merge_segments(&self.dir, &merged_name, &refs)?; // (merge_segments dropped its mmaps on return => on Windows the old ones can be unlinked) @@ -243,8 +251,15 @@ impl IndexWriter { } } - let next_gen = if cur_del_gen == -1 { 1 } else { cur_del_gen + 1 }; - let del_fname = format!("{seg_name}_{}.del", crate::zsl::segments::to_base36(next_gen as u64)); + let next_gen = if cur_del_gen == -1 { + 1 + } else { + cur_del_gen + 1 + }; + let del_fname = format!( + "{seg_name}_{}.del", + crate::zsl::segments::to_base36(next_gen as u64) + ); let del_bytes = write_del_file(max_doc, &merged); super::durability::write_atomic(&self.dir.join(&del_fname), &del_bytes)?; del_gen_overrides.insert(seg_name.clone(), next_gen); @@ -259,7 +274,11 @@ impl IndexWriter { &flushed, )?; - Ok(CommitReport { generation, segments, doc_count }) + Ok(CommitReport { + generation, + segments, + doc_count, + }) } } @@ -301,7 +320,9 @@ mod tests { /// doc with a unique term `zqxmark` in `title` (for doc_freq) + a per-index suffix. fn doc_mark(i: usize) -> WriterDoc { - WriterDoc { fields: vec![WriterField::text("title", &format!("zqxmark unique{i}"))] } + WriterDoc { + fields: vec![WriterField::text("title", &format!("zqxmark unique{i}"))], + } } #[test] @@ -317,7 +338,10 @@ mod tests { #[test] fn document_count_tracks_base_plus_flushed_plus_buffer() { let dir = temp_kb_full(); - let opts = WriterOpts { max_buffered_docs: 2, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 2, + ..WriterOpts::default() + }; let mut w = IndexWriter::open(&dir, opts).unwrap(); assert_eq!(w.document_count(), 20); // base KB live docs @@ -342,14 +366,20 @@ mod tests { let before = ZslIndex::open(&dir).unwrap().num_docs(); assert_eq!(before, 20); - let opts = WriterOpts { max_buffered_docs: 2, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 2, + ..WriterOpts::default() + }; let mut w = IndexWriter::open(&dir, opts).unwrap(); for i in 0..5 { w.add_document(doc_mark(i)).unwrap(); // cap 2 → flushes after 2 and 4; buffer=1 } let rep = w.commit().unwrap(); // flush the remaining 1 → 3 segments: _3,_4,_5 assert_eq!(rep.doc_count, 5); - assert_eq!(rep.segments, vec!["_3".to_string(), "_4".to_string(), "_5".to_string()]); + assert_eq!( + rep.segments, + vec!["_3".to_string(), "_4".to_string(), "_5".to_string()] + ); assert_eq!(rep.generation, 7); // the native reader sees base + 5 docs, the term spread across the 3 segments @@ -363,7 +393,10 @@ mod tests { #[test] fn drop_without_commit_untouched_generation_and_cleans_orphans() { let dir = temp_kb_full(); - let opts = WriterOpts { max_buffered_docs: 1, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 1, + ..WriterOpts::default() + }; { let mut w = IndexWriter::open(&dir, opts).unwrap(); w.add_document(doc_mark(0)).unwrap(); // cap 1 → immediate flush: _3.cfs @@ -373,7 +406,7 @@ mod tests { assert!(!dir.join("_3.cfs").exists()); // orphan cleaned by Drop assert!(!dir.join("segments_7").exists()); // generation NOT flipped assert_eq!(ZslIndex::open(&dir).unwrap().num_docs(), 20); // intact - // lock released → re-open OK + // lock released → re-open OK let _w = IndexWriter::open(&dir, WriterOpts::default()).unwrap(); std::fs::remove_dir_all(&dir).ok(); } @@ -429,7 +462,10 @@ mod tests { let dir = temp_kb_full(); // step 1: build a multi-segment base: _2 (KB, 20 docs) + _3,_4 (4 new docs). - let opts = WriterOpts { max_buffered_docs: 2, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 2, + ..WriterOpts::default() + }; let mut w1 = IndexWriter::open(&dir, opts).unwrap(); for i in 0..4 { w1.add_document(doc_mark(i)).unwrap(); // cap 2 → flush after 2 and after 4: _3, _4 @@ -469,7 +505,10 @@ mod tests { let dir = temp_kb_full(); // multi-seg base + deletes - let opts = WriterOpts { max_buffered_docs: 2, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 2, + ..WriterOpts::default() + }; let mut w = IndexWriter::open(&dir, opts).unwrap(); for i in 0..4 { w.add_document(doc_mark(i)).unwrap(); @@ -524,7 +563,10 @@ mod tests { let w = IndexWriter::open(&dir, WriterOpts::default()).unwrap(); // KB is 1 segment with no deletions → optimize is a no-op. let rep = w.optimize().unwrap(); - assert_eq!(rep.doc_count, before, "optimize no-op must report the index total"); + assert_eq!( + rep.doc_count, before, + "optimize no-op must report the index total" + ); std::fs::remove_dir_all(&dir).ok(); } diff --git a/sdsearch-core/src/zsl/writer/invert.rs b/sdsearch-core/src/zsl/writer/invert.rs index f5119d4..fab68c3 100644 --- a/sdsearch-core/src/zsl/writer/invert.rs +++ b/sdsearch-core/src/zsl/writer/invert.rs @@ -64,7 +64,10 @@ pub fn invert(docs: &[WriterDoc], _opts: &WriterOpts) -> Inverted { for field in &doc.fields { // field number = first-seen order (== ZSL's addField). let field_num = *field_index.entry(field.name.clone()).or_insert_with(|| { - fields.push(FieldMeta { name: field.name.clone(), indexed: false }); + fields.push(FieldMeta { + name: field.name.clone(), + indexed: false, + }); fields.len() - 1 }); // NOTE (minor divergence vs ZSL, byte-diagnostic-only): ZSL clones a Text/Keyword @@ -140,14 +143,22 @@ pub fn invert(docs: &[WriterDoc], _opts: &WriterOpts) -> Inverted { .enumerate() .map(|(fnum, meta)| { if meta.indexed { - (0..doc_count).map(|d| field_doc_numterms.get(&(fnum, d)).copied()).collect() + (0..doc_count) + .map(|d| field_doc_numterms.get(&(fnum, d)).copied()) + .collect() } else { Vec::new() } }) .collect(); - Inverted { fields, terms, norm_lengths, stored, doc_count } + Inverted { + fields, + terms, + norm_lengths, + stored, + doc_count, + } } #[cfg(test)] @@ -199,10 +210,19 @@ mod tests { let body = 1; // term order by (fieldName·\0·text): body\0new < title\0done < title\0new < title\0workflow - let order: Vec<(usize, &str)> = inv.terms.iter().map(|t| (t.field_num, t.text.as_str())).collect(); + let order: Vec<(usize, &str)> = inv + .terms + .iter() + .map(|t| (t.field_num, t.text.as_str())) + .collect(); assert_eq!( order, - vec![(body, "new"), (title, "done"), (title, "new"), (title, "workflow")] + vec![ + (body, "new"), + (title, "done"), + (title, "new"), + (title, "workflow") + ] ); // 1-based positions within the field @@ -269,8 +289,16 @@ mod tests { assert_eq!( inv.stored[0], vec![ - StoredField { field_num: 0, value: "hi".into(), tokenized: true }, - StoredField { field_num: 1, value: "999".into(), tokenized: false }, + StoredField { + field_num: 0, + value: "hi".into(), + tokenized: true + }, + StoredField { + field_num: 1, + value: "999".into(), + tokenized: false + }, ] ); } diff --git a/sdsearch-core/src/zsl/writer/merge.rs b/sdsearch-core/src/zsl/writer/merge.rs index 1d99b2a..898bec4 100644 --- a/sdsearch-core/src/zsl/writer/merge.rs +++ b/sdsearch-core/src/zsl/writer/merge.rs @@ -45,7 +45,10 @@ pub fn merge_segments( for seg in &segs { for fi in seg.field_infos() { let idx = *field_index.entry(fi.name.clone()).or_insert_with(|| { - fields.push(FieldMeta { name: fi.name.clone(), indexed: false }); + fields.push(FieldMeta { + name: fi.name.clone(), + indexed: false, + }); fields.len() - 1 }); fields[idx].indexed |= fi.is_indexed; @@ -78,15 +81,20 @@ pub fn merge_segments( if r.is_binary { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!("merge: binary stored field not supported (local field_num {})", r.field_num), + format!( + "merge: binary stored field not supported (local field_num {})", + r.field_num + ), )); } let name = &local_fields .get(r.field_num) - .ok_or_else(|| std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("merge: stored field_num {} out of range", r.field_num), - ))? + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("merge: stored field_num {} out of range", r.field_num), + ) + })? .name; remapped.push(StoredField { field_num: field_index[name], @@ -166,7 +174,10 @@ pub fn merge_segments( let dict = terms::write_term_dict(&merged_terms); let cfs_bytes = assemble_cfs(merged_name, &fnm_bytes, &fdt, &fdx, &nrm, &dict); - Ok(MergeResult { cfs_bytes, doc_count }) + Ok(MergeResult { + cfs_bytes, + doc_count, + }) } #[cfg(test)] @@ -197,13 +208,16 @@ mod tests { } fn doc_mark(i: usize) -> WriterDoc { - WriterDoc { fields: vec![WriterField::text("title", &format!("zqxmark unique{i}"))] } + WriterDoc { + fields: vec![WriterField::text("title", &format!("zqxmark unique{i}"))], + } } /// writes the merged bytes as the ONLY segment in a new dir and opens it with the reader. fn read_merged(cfs_bytes: &[u8], doc_count: usize) -> ZslSegment { let n = COUNTER.fetch_add(1, Ordering::Relaxed); - let out = std::env::temp_dir().join(format!("sdsearch_merged_{}_{}", std::process::id(), n)); + let out = + std::env::temp_dir().join(format!("sdsearch_merged_{}_{}", std::process::id(), n)); std::fs::create_dir_all(&out).unwrap(); std::fs::write(out.join("_m.cfs"), cfs_bytes).unwrap(); let seg = ZslSegment::open_named(&out, "_m", -1).unwrap(); @@ -217,7 +231,10 @@ mod tests { let dir = temp_kb_full(); // multi-segment base: KB _2 (20 docs) + _3,_4 (4 docs with unique term 'zqxmark') - let opts = WriterOpts { max_buffered_docs: 2, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 2, + ..WriterOpts::default() + }; let mut w = IndexWriter::open(&dir, opts).unwrap(); for i in 0..4 { w.add_document(doc_mark(i)).unwrap(); @@ -274,7 +291,10 @@ mod tests { std::env::temp_dir().join(format!("sdsearch_merge_bin_{}_{}", std::process::id(), n)); std::fs::create_dir_all(&dir).unwrap(); - let fields = vec![FieldMeta { name: "bin_field".to_string(), indexed: false }]; + let fields = vec![FieldMeta { + name: "bin_field".to_string(), + indexed: false, + }]; let fnm_bytes = fnm::write_fnm(&fields); // .fdt of a doc with one field: VInt(stored_count=1) + VInt(field_num=0) + flags(0x02) @@ -323,7 +343,10 @@ mod tests { std::env::temp_dir().join(format!("sdsearch_merge_noprx_{}_{}", std::process::id(), n)); std::fs::create_dir_all(&dir).unwrap(); - let fields = vec![FieldMeta { name: "body".to_string(), indexed: true }]; + let fields = vec![FieldMeta { + name: "body".to_string(), + indexed: true, + }]; let fnm_bytes = fnm::write_fnm(&fields); // a doc with no stored fields: enough for `open_named` to compute max_doc == 1. @@ -360,12 +383,19 @@ mod tests { let dir = temp_kb_full(); // cap=1 → each add flushes its own segment: KB (_2) + 2 new ones (_3, _4). - let opts = WriterOpts { max_buffered_docs: 1, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 1, + ..WriterOpts::default() + }; let mut w = IndexWriter::open(&dir, opts).unwrap(); - w.add_document(WriterDoc { fields: vec![WriterField::text("title", "zqmerge alpha")] }) - .unwrap(); - w.add_document(WriterDoc { fields: vec![WriterField::text("title", "zqmerge beta")] }) - .unwrap(); + w.add_document(WriterDoc { + fields: vec![WriterField::text("title", "zqmerge alpha")], + }) + .unwrap(); + w.add_document(WriterDoc { + fields: vec![WriterField::text("title", "zqmerge beta")], + }) + .unwrap(); w.commit().unwrap(); // + a delete on the base (gid 0, lives in _2): the merge must EXCLUDE that doc, not @@ -375,7 +405,11 @@ mod tests { w2.commit().unwrap(); let infos = read_segment_infos(&dir).unwrap(); - assert!(infos.len() >= 3, "expected multi-segment (KB + 2 flushed), got {}", infos.len()); + assert!( + infos.len() >= 3, + "expected multi-segment (KB + 2 flushed), got {}", + infos.len() + ); let live = ZslIndex::open(&dir).unwrap().num_docs(); // DIRECT merge of the primitive (not via `IndexWriter::optimize()`) over all the @@ -384,7 +418,10 @@ mod tests { let gen = crate::zsl::writer::segments::read_generation(&dir).unwrap(); let merged_name = crate::zsl::writer::segments::segment_name(gen.name_counter); let merged = merge_segments(&dir, &merged_name, &refs).unwrap(); - assert_eq!(merged.doc_count, live, "merge doc_count must equal the total live docs"); + assert_eq!( + merged.doc_count, live, + "merge doc_count must equal the total live docs" + ); // write the merged `.cfs` and flip the generation by hand (same steps 4-5 as // `optimize()`, but exercising `merge_segments` directly) to re-read with the full diff --git a/sdsearch-core/src/zsl/writer/mod.rs b/sdsearch-core/src/zsl/writer/mod.rs index 265630a..67af6ee 100644 --- a/sdsearch-core/src/zsl/writer/mod.rs +++ b/sdsearch-core/src/zsl/writer/mod.rs @@ -50,13 +50,28 @@ pub struct WriterField { impl WriterField { pub fn text(name: &str, value: &str) -> Self { - Self { name: name.into(), value: value.into(), kind: FieldKind::Text, stored: true } + Self { + name: name.into(), + value: value.into(), + kind: FieldKind::Text, + stored: true, + } } pub fn keyword(name: &str, value: &str) -> Self { - Self { name: name.into(), value: value.into(), kind: FieldKind::Keyword, stored: true } + Self { + name: name.into(), + value: value.into(), + kind: FieldKind::Keyword, + stored: true, + } } pub fn unindexed(name: &str, value: &str) -> Self { - Self { name: name.into(), value: value.into(), kind: FieldKind::UnIndexed, stored: true } + Self { + name: name.into(), + value: value.into(), + kind: FieldKind::UnIndexed, + stored: true, + } } } @@ -76,7 +91,10 @@ pub struct WriterOpts { impl Default for WriterOpts { fn default() -> Self { - Self { doc_boost: 1.0, max_buffered_docs: 1000 } + Self { + doc_boost: 1.0, + max_buffered_docs: 1000, + } } } @@ -157,9 +175,14 @@ pub fn append_documents( let doc_count = write_segment_cfs(index_dir, &seg_name, docs, opts)?; // only once the .cfs is on disk do we flip the generation. - let new_gen = segments::write_appended_generation(index_dir, &gen, &seg_name, doc_count as u32)?; + let new_gen = + segments::write_appended_generation(index_dir, &gen, &seg_name, doc_count as u32)?; - Ok(AppendReport { segment_name: seg_name, doc_count, generation: new_gen }) + Ok(AppendReport { + segment_name: seg_name, + doc_count, + generation: new_gen, + }) } #[cfg(test)] @@ -174,7 +197,8 @@ mod tests { /// copies the ENTIRE KB fixture (incl. `_2.cfs`) to a fresh temp dir. fn temp_kb_full() -> std::path::PathBuf { let n = COUNTER.fetch_add(1, Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!("sdsearch_append_{}_{}", std::process::id(), n)); + let dir = + std::env::temp_dir().join(format!("sdsearch_append_{}_{}", std::process::id(), n)); std::fs::create_dir_all(&dir).unwrap(); let src = std::path::PathBuf::from(concat!( env!("CARGO_MANIFEST_DIR"), diff --git a/sdsearch-core/src/zsl/writer/postings.rs b/sdsearch-core/src/zsl/writer/postings.rs index f477d18..e271609 100644 --- a/sdsearch-core/src/zsl/writer/postings.rs +++ b/sdsearch-core/src/zsl/writer/postings.rs @@ -52,9 +52,16 @@ mod tests { let (fp, pp) = write_term_postings(&mut frq, &mut prx, &docs); assert_eq!((fp, pp), (0, 0)); - let info = TermInfo { doc_freq: 2, freq_pointer: fp, prox_pointer: pp }; + let info = TermInfo { + doc_freq: 2, + freq_pointer: fp, + prox_pointer: pp, + }; assert_eq!(read_freqs(&frq, &info).unwrap(), vec![(0, 2), (2, 1)]); - assert_eq!(read_all_positions(&frq, &prx, &info).unwrap(), vec![(0, vec![1, 2]), (2, vec![1])]); + assert_eq!( + read_all_positions(&frq, &prx, &info).unwrap(), + vec![(0, vec![1, 2]), (2, vec![1])] + ); } #[test] @@ -66,10 +73,17 @@ mod tests { let (fp, pp) = write_term_postings(&mut frq, &mut prx, &[(1usize, vec![3u32, 7])]); assert!(fp > 0 && pp > 0); - let info = TermInfo { doc_freq: 1, freq_pointer: fp, prox_pointer: pp }; + let info = TermInfo { + doc_freq: 1, + freq_pointer: fp, + prox_pointer: pp, + }; assert_eq!(read_freqs(&frq, &info).unwrap(), vec![(1, 2)]); // 0-based position? no: we store them raw; deltas 3 then 4 -> 3,7 - assert_eq!(read_all_positions(&frq, &prx, &info).unwrap(), vec![(1, vec![3, 7])]); + assert_eq!( + read_all_positions(&frq, &prx, &info).unwrap(), + vec![(1, vec![3, 7])] + ); } #[test] @@ -78,8 +92,15 @@ mod tests { let mut frq = Vec::new(); let mut prx = Vec::new(); let (fp, pp) = write_term_postings(&mut frq, &mut prx, &[(5usize, vec![0u32])]); - let info = TermInfo { doc_freq: 1, freq_pointer: fp, prox_pointer: pp }; + let info = TermInfo { + doc_freq: 1, + freq_pointer: fp, + prox_pointer: pp, + }; assert_eq!(read_freqs(&frq, &info).unwrap(), vec![(5, 1)]); - assert_eq!(read_all_positions(&frq, &prx, &info).unwrap(), vec![(5, vec![0])]); + assert_eq!( + read_all_positions(&frq, &prx, &info).unwrap(), + vec![(5, vec![0])] + ); } } diff --git a/sdsearch-core/src/zsl/writer/segments.rs b/sdsearch-core/src/zsl/writer/segments.rs index 063b2cb..909ab18 100644 --- a/sdsearch-core/src/zsl/writer/segments.rs +++ b/sdsearch-core/src/zsl/writer/segments.rs @@ -143,7 +143,10 @@ pub fn write_appended_generation( write_generation_with_new_segments( index_dir, gen, - &[NewSegment { name: new_segment_name.to_string(), doc_count: new_segment_doc_count }], + &[NewSegment { + name: new_segment_name.to_string(), + doc_count: new_segment_doc_count, + }], ) } @@ -154,7 +157,12 @@ pub fn write_generation_with_new_segments( gen: &Generation, new_segments: &[NewSegment], ) -> std::io::Result { - write_generation_with_delgens(index_dir, gen, &std::collections::HashMap::new(), new_segments) + write_generation_with_delgens( + index_dir, + gen, + &std::collections::HashMap::new(), + new_segments, + ) } /// Writes the next generation: copies the existing records verbatim BUT patches in-place the @@ -327,8 +335,14 @@ mod tests { &dir, &gen, &[ - NewSegment { name: "_3".into(), doc_count: 3 }, - NewSegment { name: "_4".into(), doc_count: 4 }, + NewSegment { + name: "_3".into(), + doc_count: 3, + }, + NewSegment { + name: "_4".into(), + doc_count: 4, + }, ], ) .unwrap(); diff --git a/sdsearch-core/src/zsl/writer/stored.rs b/sdsearch-core/src/zsl/writer/stored.rs index 33a65c3..74aa624 100644 --- a/sdsearch-core/src/zsl/writer/stored.rs +++ b/sdsearch-core/src/zsl/writer/stored.rs @@ -33,15 +33,33 @@ mod tests { fn stored_roundtrips_through_reader_preserving_trailing_newline() { let stored = vec![ vec![ - StoredField { field_num: 0, value: "New workflow\n".into(), tokenized: true }, - StoredField { field_num: 1, value: "42".into(), tokenized: false }, + StoredField { + field_num: 0, + value: "New workflow\n".into(), + tokenized: true, + }, + StoredField { + field_num: 1, + value: "42".into(), + tokenized: false, + }, ], - vec![StoredField { field_num: 0, value: "other\n".into(), tokenized: true }], + vec![StoredField { + field_num: 0, + value: "other\n".into(), + tokenized: true, + }], ]; let (fdt, fdx) = write_stored(&stored); let fields = vec![ - FieldInfo { name: "title".into(), is_indexed: true }, - FieldInfo { name: "id".into(), is_indexed: false }, + FieldInfo { + name: "title".into(), + is_indexed: true, + }, + FieldInfo { + name: "id".into(), + is_indexed: false, + }, ]; let d0 = read_stored_fields(&fdx, &fdt, &fields, 0).unwrap(); @@ -58,25 +76,55 @@ mod tests { use crate::zsl::stored::{read_stored_raw, StoredRaw}; let stored = vec![ vec![ - StoredField { field_num: 0, value: "New workflow\n".into(), tokenized: true }, - StoredField { field_num: 2, value: "42".into(), tokenized: false }, + StoredField { + field_num: 0, + value: "New workflow\n".into(), + tokenized: true, + }, + StoredField { + field_num: 2, + value: "42".into(), + tokenized: false, + }, ], - vec![StoredField { field_num: 0, value: "other\n".into(), tokenized: true }], + vec![StoredField { + field_num: 0, + value: "other\n".into(), + tokenized: true, + }], ]; let (fdt, fdx) = write_stored(&stored); // doc 0: preserves order, field_num and the tokenized flag assert_eq!( read_stored_raw(&fdx, &fdt, 0).unwrap(), vec![ - StoredRaw { field_num: 0, value: "New workflow\n".into(), tokenized: true, is_binary: false }, - StoredRaw { field_num: 2, value: "42".into(), tokenized: false, is_binary: false }, + StoredRaw { + field_num: 0, + value: "New workflow\n".into(), + tokenized: true, + is_binary: false + }, + StoredRaw { + field_num: 2, + value: "42".into(), + tokenized: false, + is_binary: false + }, ] ); // the host application does not index binaries: the round-trip must never report is_binary=true - assert!(read_stored_raw(&fdx, &fdt, 0).unwrap().iter().all(|r| !r.is_binary)); + assert!(read_stored_raw(&fdx, &fdt, 0) + .unwrap() + .iter() + .all(|r| !r.is_binary)); assert_eq!( read_stored_raw(&fdx, &fdt, 1).unwrap(), - vec![StoredRaw { field_num: 0, value: "other\n".into(), tokenized: true, is_binary: false }] + vec![StoredRaw { + field_num: 0, + value: "other\n".into(), + tokenized: true, + is_binary: false + }] ); // out-of-range doc → empty assert!(read_stored_raw(&fdx, &fdt, 9).unwrap().is_empty()); diff --git a/sdsearch-core/src/zsl/writer/terms.rs b/sdsearch-core/src/zsl/writer/terms.rs index 580e63f..0c68fe4 100644 --- a/sdsearch-core/src/zsl/writer/terms.rs +++ b/sdsearch-core/src/zsl/writer/terms.rs @@ -47,7 +47,7 @@ pub fn write_term_dict(terms: &[TermPostings]) -> DictFiles { // state of the previous term in the .tis (borrows term text from `terms`, no clone) let mut prev: Option<(&str, usize, u64, u64)> = None; // (text, field, freqPtr, proxPtr) - // state of the last sample in the .tii + // state of the last sample in the .tii let mut idx_prev: Option<(&str, usize, u64, u64)> = None; let mut last_index_position: u64 = 24; @@ -176,12 +176,17 @@ mod tests { assert_eq!(read_freqs(&f.frq, body_new).unwrap(), vec![(0, 2)]); let title_wf = dict.info("title", "workflow").unwrap(); - assert_eq!(read_all_positions(&f.frq, &f.prx, title_wf).unwrap(), vec![(0, vec![2]), (1, vec![1])]); + assert_eq!( + read_all_positions(&f.frq, &f.prx, title_wf).unwrap(), + vec![(0, vec![2]), (1, vec![1])] + ); } #[test] fn tis_header_has_marker_and_backpatched_term_count() { - let docs = vec![WriterDoc { fields: vec![WriterField::text("t", "a b c")] }]; + let docs = vec![WriterDoc { + fields: vec![WriterField::text("t", "a b c")], + }]; let inv = invert(&docs, &WriterOpts::default()); let f = write_term_dict(&inv.terms); assert_eq!(&f.tis[0..4], &[0xFF, 0xFF, 0xFF, 0xFD]); // marker @@ -191,14 +196,18 @@ mod tests { #[test] fn tii_starts_with_synthetic_entry_and_count_one_for_small_batch() { - let docs = vec![WriterDoc { fields: vec![WriterField::text("t", "a b c")] }]; + let docs = vec![WriterDoc { + fields: vec![WriterField::text("t", "a b c")], + }]; let inv = invert(&docs, &WriterOpts::default()); let f = write_term_dict(&inv.terms); // header 24 bytes; count == 1 (fewer than indexInterval terms) let mut pos = 4; assert_eq!(read_u64_be(&f.tii, &mut pos).unwrap(), 1); // synthetic entry: VInt(0) VInt(0) Int(0xFFFFFFFF) byte(0x0F) VInt0 VInt0 VInt0 VInt(24) - let expected = [0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F, 0x00, 0x00, 0x00, 0x18]; + let expected = [ + 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F, 0x00, 0x00, 0x00, 0x18, + ]; assert_eq!(&f.tii[24..24 + expected.len()], &expected); assert_eq!(f.tii.len(), 24 + expected.len()); // only header + synthetic } @@ -206,8 +215,13 @@ mod tests { #[test] fn tii_samples_every_index_interval_terms() { // 300 unique terms in one field -> count = (300 - 300%128)/128 + 1 = 3 - let value: String = (0..300).map(|i| format!("w{i:04}")).collect::>().join(" "); - let docs = vec![WriterDoc { fields: vec![WriterField::text("t", &value)] }]; + let value: String = (0..300) + .map(|i| format!("w{i:04}")) + .collect::>() + .join(" "); + let docs = vec![WriterDoc { + fields: vec![WriterField::text("t", &value)], + }]; let inv = invert(&docs, &WriterOpts::default()); assert_eq!(inv.terms.len(), 300); let f = write_term_dict(&inv.terms); diff --git a/sdsearch-core/tests/persistence_roundtrip.rs b/sdsearch-core/tests/persistence_roundtrip.rs index a707458..027ed1d 100644 --- a/sdsearch-core/tests/persistence_roundtrip.rs +++ b/sdsearch-core/tests/persistence_roundtrip.rs @@ -2,7 +2,9 @@ use sdsearch_core::doc::{Document, FieldKind}; use sdsearch_core::index::MemoryIndex; -use sdsearch_core::search::{fuzzy_query, multi_term_query, phrase_query, term_query, wildcard_query}; +use sdsearch_core::search::{ + fuzzy_query, multi_term_query, phrase_query, term_query, wildcard_query, +}; use sdsearch_core::segment::Segment; fn corpus() -> MemoryIndex { @@ -37,7 +39,12 @@ fn term_and_multiterm_match_between_memory_and_disk() { let b = term_query(&seg, "body", "fox", 0.0, 10); assert_eq!(ids(&a), ids(&b)); for (x, y) in a.iter().zip(b.iter()) { - assert!((x.score - y.score).abs() < 1e-6, "score mismatch {} vs {}", x.score, y.score); + assert!( + (x.score - y.score).abs() < 1e-6, + "score mismatch {} vs {}", + x.score, + y.score + ); assert_eq!(x.fields, y.fields); } diff --git a/sdsearch-core/tests/zsl_boolean_parity.rs b/sdsearch-core/tests/zsl_boolean_parity.rs index e6ded7b..1f30265 100644 --- a/sdsearch-core/tests/zsl_boolean_parity.rs +++ b/sdsearch-core/tests/zsl_boolean_parity.rs @@ -7,24 +7,39 @@ use std::collections::HashMap; use std::path::PathBuf; #[derive(serde::Deserialize)] -struct Expected { bool_queries: HashMap> } +struct Expected { + bool_queries: HashMap>, +} #[derive(serde::Deserialize)] -struct HitId { id: usize } +struct HitId { + id: usize, +} fn expected() -> Expected { let raw = std::fs::read_to_string(concat!( - env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_expected_multiseg.json" - )).expect("multiseg oracle missing"); + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_expected_multiseg.json" + )) + .expect("multiseg oracle missing"); serde_json::from_str(&raw).unwrap() } fn idx() -> ZslIndex { ZslIndex::open(&PathBuf::from(concat!( - env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_multiseg" - ))).unwrap() + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_multiseg" + ))) + .unwrap() +} +fn sorted(mut v: Vec) -> Vec { + v.sort(); + v +} +fn eng(hits: &[Hit]) -> Vec { + sorted(hits.iter().map(|h| h.id).collect()) +} +fn ora(e: &Expected, k: &str) -> Vec { + sorted(e.bool_queries[k].iter().map(|h| h.id).collect()) } -fn sorted(mut v: Vec) -> Vec { v.sort(); v } -fn eng(hits: &[Hit]) -> Vec { sorted(hits.iter().map(|h| h.id).collect()) } -fn ora(e: &Expected, k: &str) -> Vec { sorted(e.bool_queries[k].iter().map(|h| h.id).collect()) } fn base(text: &str) -> QueryParams { QueryParams { @@ -48,18 +63,31 @@ fn text_only_matches_oracle() { fn text_plus_where_matches_oracle() { let (i, e) = (idx(), expected()); let mut p = base("vpn"); - p.where_groups = vec![WhereGroup { field: "lang".into(), values: vec!["es".into()], occur: Occur::Should }]; + p.where_groups = vec![WhereGroup { + field: "lang".into(), + values: vec!["es".into()], + occur: Occur::Should, + }]; let q = build_query(&p).unwrap(); - assert_eq!(eng(&search(&i, &q, 0.0, 100)), ora(&e, "text+where:vpn|lang=es")); + assert_eq!( + eng(&search(&i, &q, 0.0, 100)), + ora(&e, "text+where:vpn|lang=es") + ); } #[test] fn text_plus_in_matches_oracle() { let (i, e) = (idx(), expected()); let mut p = base("vpn"); - p.in_groups = vec![InGroup { field: "cat".into(), values: vec!["1".into(), "2".into()] }]; + p.in_groups = vec![InGroup { + field: "cat".into(), + values: vec!["1".into(), "2".into()], + }]; let q = build_query(&p).unwrap(); - assert_eq!(eng(&search(&i, &q, 0.0, 100)), ora(&e, "text+in:vpn|cat=1,2")); + assert_eq!( + eng(&search(&i, &q, 0.0, 100)), + ora(&e, "text+in:vpn|cat=1,2") + ); } #[test] @@ -69,20 +97,36 @@ fn text_plus_multi_in_matches_oracle() { let (i, e) = (idx(), expected()); let mut p = base("vpn"); p.in_groups = vec![ - InGroup { field: "cat".into(), values: vec!["3".into()] }, - InGroup { field: "lang".into(), values: vec!["en".into()] }, + InGroup { + field: "cat".into(), + values: vec!["3".into()], + }, + InGroup { + field: "lang".into(), + values: vec!["en".into()], + }, ]; let q = build_query(&p).unwrap(); - assert_eq!(eng(&search(&i, &q, 0.0, 100)), ora(&e, "text+in-multi:vpn|cat=3&lang=en")); + assert_eq!( + eng(&search(&i, &q, 0.0, 100)), + ora(&e, "text+in-multi:vpn|cat=3&lang=en") + ); } #[test] fn where_mustnot_matches_oracle() { let (i, e) = (idx(), expected()); let mut p = base("how"); - p.where_groups = vec![WhereGroup { field: "lang".into(), values: vec!["en".into()], occur: Occur::MustNot }]; + p.where_groups = vec![WhereGroup { + field: "lang".into(), + values: vec!["en".into()], + occur: Occur::MustNot, + }]; let q = build_query(&p).unwrap(); - assert_eq!(eng(&search(&i, &q, 0.0, 100)), ora(&e, "where-mustnot:how|lang!=en")); + assert_eq!( + eng(&search(&i, &q, 0.0, 100)), + ora(&e, "where-mustnot:how|lang!=en") + ); } #[test] @@ -90,5 +134,8 @@ fn text_multiword_matches_oracle() { // multi-word coverage of free text (the previous oracle only had single-word cases). let (i, e) = (idx(), expected()); let q = build_query(&base("how to")).unwrap(); - assert_eq!(eng(&search(&i, &q, 0.0, 100)), ora(&e, "text-multiword:how to")); + assert_eq!( + eng(&search(&i, &q, 0.0, 100)), + ora(&e, "text-multiword:how to") + ); } diff --git a/sdsearch-core/tests/zsl_interprocess_lock.rs b/sdsearch-core/tests/zsl_interprocess_lock.rs index d5aba64..cdbdfa5 100644 --- a/sdsearch-core/tests/zsl_interprocess_lock.rs +++ b/sdsearch-core/tests/zsl_interprocess_lock.rs @@ -13,11 +13,16 @@ fn temp_kb() -> PathBuf { let n = COUNTER.fetch_add(1, Ordering::Relaxed); let dir = std::env::temp_dir().join(format!("sdsearch_iplock_{}_{}", std::process::id(), n)); std::fs::create_dir_all(&dir).unwrap(); - let src = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_kb")); + let src = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_kb" + )); for f in std::fs::read_dir(&src).unwrap() { let f = f.unwrap().path(); let name = f.file_name().unwrap().to_string_lossy(); - if name.contains("lock") || name.ends_with(".sti") { continue; } + if name.contains("lock") || name.ends_with(".sti") { + continue; + } std::fs::copy(&f, dir.join(f.file_name().unwrap())).unwrap(); } dir @@ -28,15 +33,24 @@ fn example_bin() -> PathBuf { // CARGO_BIN_EXE_ does not apply to examples; use the test exe's dir. let mut p = std::env::current_exe().unwrap(); p.pop(); // .../deps - if p.ends_with("deps") { p.pop(); } - p.join(if cfg!(windows) { "examples\\stream_writer.exe" } else { "examples/stream_writer" }) + if p.ends_with("deps") { + p.pop(); + } + p.join(if cfg!(windows) { + "examples\\stream_writer.exe" + } else { + "examples/stream_writer" + }) } #[test] fn write_lock_excludes_a_second_process() { let dir = temp_kb(); let bin = example_bin(); - assert!(bin.is_file(), "build the example: cargo build -p sdsearch-core --example stream_writer"); + assert!( + bin.is_file(), + "build the example: cargo build -p sdsearch-core --example stream_writer" + ); // Process A: takes the lock and holds it for 1500 ms. let mut a = Command::new(&bin) @@ -49,8 +63,10 @@ fn write_lock_excludes_a_second_process() { let mut a_out = a.stdout.take().unwrap(); let mut buf = [0u8; 32]; let nread = a_out.read(&mut buf).unwrap(); - assert!(String::from_utf8_lossy(&buf[..nread]).contains("LOCK_ACQUIRED"), - "A did not take the lock"); + assert!( + String::from_utf8_lossy(&buf[..nread]).contains("LOCK_ACQUIRED"), + "A did not take the lock" + ); // Process B: tries to take the lock while A holds it → must fail with exit 3. let b = Command::new(&bin) @@ -58,8 +74,12 @@ fn write_lock_excludes_a_second_process() { .stdout(Stdio::piped()) .output() .unwrap(); - assert_eq!(b.status.code(), Some(3), - "B should receive WouldBlock (exit 3), stdout={}", String::from_utf8_lossy(&b.stdout)); + assert_eq!( + b.status.code(), + Some(3), + "B should receive WouldBlock (exit 3), stdout={}", + String::from_utf8_lossy(&b.stdout) + ); assert!(String::from_utf8_lossy(&b.stdout).contains("LOCK_WOULDBLOCK")); // A releases on exit; then C must be able to take it. @@ -69,7 +89,11 @@ fn write_lock_excludes_a_second_process() { .stdout(Stdio::piped()) .output() .unwrap(); - assert_eq!(c.status.code(), Some(0), "C should take the lock after A releases it"); + assert_eq!( + c.status.code(), + Some(0), + "C should take the lock after A releases it" + ); assert!(String::from_utf8_lossy(&c.stdout).contains("LOCK_ACQUIRED")); std::fs::remove_dir_all(&dir).ok(); diff --git a/sdsearch-core/tests/zsl_multiseg_parity.rs b/sdsearch-core/tests/zsl_multiseg_parity.rs index 1ce3707..5714a29 100644 --- a/sdsearch-core/tests/zsl_multiseg_parity.rs +++ b/sdsearch-core/tests/zsl_multiseg_parity.rs @@ -13,27 +13,47 @@ struct Expected { docs: Vec, } #[derive(serde::Deserialize)] -struct HitId { id: usize } +struct HitId { + id: usize, +} #[derive(serde::Deserialize)] -struct DocEntry { id: usize, stored: HashMap } +struct DocEntry { + id: usize, + stored: HashMap, +} fn expected() -> Expected { let raw = std::fs::read_to_string(concat!( - env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_expected_multiseg.json" - )).expect("multiseg oracle missing — run tools/gen_zsl_multiseg_fixture.php"); + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_expected_multiseg.json" + )) + .expect("multiseg oracle missing — run tools/gen_zsl_multiseg_fixture.php"); serde_json::from_str(&raw).unwrap() } fn idx() -> ZslIndex { ZslIndex::open(&PathBuf::from(concat!( - env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_multiseg" - ))).unwrap() + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_multiseg" + ))) + .unwrap() +} +fn sorted(mut v: Vec) -> Vec { + v.sort(); + v +} +fn eng(hits: &[Hit]) -> Vec { + sorted(hits.iter().map(|h| h.id).collect()) +} +fn ora(e: &Expected, k: &str) -> Vec { + sorted(e.queries[k].iter().map(|h| h.id).collect()) } -fn sorted(mut v: Vec) -> Vec { v.sort(); v } -fn eng(hits: &[Hit]) -> Vec { sorted(hits.iter().map(|h| h.id).collect()) } -fn ora(e: &Expected, k: &str) -> Vec { sorted(e.queries[k].iter().map(|h| h.id).collect()) } // RAW order (unsorted) exactly as the engine / oracle returns it — to check ranking coherence. -fn eng_raw(hits: &[Hit]) -> Vec { hits.iter().map(|h| h.id).collect() } -fn ora_raw(e: &Expected, k: &str) -> Vec { e.queries[k].iter().map(|h| h.id).collect() } +fn eng_raw(hits: &[Hit]) -> Vec { + hits.iter().map(|h| h.id).collect() +} +fn ora_raw(e: &Expected, k: &str) -> Vec { + e.queries[k].iter().map(|h| h.id).collect() +} #[test] fn num_docs_excludes_deleted() { @@ -47,7 +67,10 @@ fn term_queries_cross_segment_boundary() { let vpn = ora(&e, "term:title:vpn"); assert_eq!(eng(&term_query(&i, "title", "vpn", 0.0, 100)), vpn); assert!(vpn.len() >= 2, "vpn must cross segments"); - assert_eq!(eng(&term_query(&i, "title", "mysql", 0.0, 100)), ora(&e, "term:title:mysql")); + assert_eq!( + eng(&term_query(&i, "title", "mysql", 0.0, 100)), + ora(&e, "term:title:mysql") + ); } #[test] @@ -56,22 +79,37 @@ fn term_vpn_rank_order_matches_oracle() { // so the engine's score-desc/id-asc tie-break must match ZSL's raw order. let (i, e) = (idx(), expected()); let expected_order = ora_raw(&e, "term:title:vpn"); - assert_eq!(eng_raw(&term_query(&i, "title", "vpn", 0.0, 100)), expected_order); + assert_eq!( + eng_raw(&term_query(&i, "title", "vpn", 0.0, 100)), + expected_order + ); } #[test] fn deleted_doc_term_returns_empty() { let (i, e) = (idx(), expected()); assert_eq!(ora(&e, "term:title:backup"), Vec::::new()); - assert_eq!(eng(&term_query(&i, "title", "backup", 0.0, 100)), Vec::::new()); + assert_eq!( + eng(&term_query(&i, "title", "backup", 0.0, 100)), + Vec::::new() + ); } #[test] fn wildcard_fuzzy_phrase_match_oracle() { let (i, e) = (idx(), expected()); - assert_eq!(eng(&wildcard_query(&i, "title", "re*", 0, 0.0, 100)), ora(&e, "wildcard:title:re*")); - assert_eq!(eng(&fuzzy_query(&i, "title", "mysgl", 0.5, 3, 0.0, 100)), ora(&e, "fuzzy:title:mysgl:0.5:3")); - assert_eq!(eng(&phrase_query(&i, "title", &["how", "to"], 0.0, 100)), ora(&e, "phrase:title:how to")); + assert_eq!( + eng(&wildcard_query(&i, "title", "re*", 0, 0.0, 100)), + ora(&e, "wildcard:title:re*") + ); + assert_eq!( + eng(&fuzzy_query(&i, "title", "mysgl", 0.5, 3, 0.0, 100)), + ora(&e, "fuzzy:title:mysgl:0.5:3") + ); + assert_eq!( + eng(&phrase_query(&i, "title", &["how", "to"], 0.0, 100)), + ora(&e, "phrase:title:how to") + ); } #[test] @@ -80,6 +118,11 @@ fn stored_routing_by_base_is_correct() { // each global id from the oracle must return the same stored id_key for d in &e.docs { let got = i.stored_fields(d.id); - assert_eq!(got.get("id_key"), d.stored.get("id_key"), "id {} mal ruteado", d.id); + assert_eq!( + got.get("id_key"), + d.stored.get("id_key"), + "id {} mal ruteado", + d.id + ); } } diff --git a/sdsearch-core/tests/zsl_optimize_crash_safety.rs b/sdsearch-core/tests/zsl_optimize_crash_safety.rs index b6dbf9e..824bf1e 100644 --- a/sdsearch-core/tests/zsl_optimize_crash_safety.rs +++ b/sdsearch-core/tests/zsl_optimize_crash_safety.rs @@ -26,7 +26,9 @@ fn temp_kb_full() -> std::path::PathBuf { } fn doc(i: usize) -> WriterDoc { - WriterDoc { fields: vec![WriterField::text("title", &format!("zqxc unique{i}"))] } + WriterDoc { + fields: vec![WriterField::text("title", &format!("zqxc unique{i}"))], + } } /// lowercase base36 (== ZSL) — replicated here because `to_base36` is internal to the crate. @@ -49,7 +51,10 @@ fn interrupted_before_gen_flip_reader_keeps_old_generation() { let dir = temp_kb_full(); // multi-seg base (gen advances via the commit) - let opts = WriterOpts { max_buffered_docs: 2, ..WriterOpts::default() }; + let opts = WriterOpts { + max_buffered_docs: 2, + ..WriterOpts::default() + }; let mut w = IndexWriter::open(&dir, opts).unwrap(); for i in 0..4 { w.add_document(doc(i)).unwrap(); diff --git a/sdsearch-core/tests/zsl_query_parity.rs b/sdsearch-core/tests/zsl_query_parity.rs index 16133e8..85d48e7 100644 --- a/sdsearch-core/tests/zsl_query_parity.rs +++ b/sdsearch-core/tests/zsl_query_parity.rs @@ -23,7 +23,11 @@ fn expected() -> Expected { } fn seg() -> ZslSegment { - ZslSegment::open(&PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index"))).unwrap() + ZslSegment::open(&PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index" + ))) + .unwrap() } fn doc_set(hits: &[sdsearch_core::search::Hit]) -> Vec { diff --git a/sdsearch-core/tests/zsl_query_parity_kb.rs b/sdsearch-core/tests/zsl_query_parity_kb.rs index 602cc3d..b6bb9c8 100644 --- a/sdsearch-core/tests/zsl_query_parity_kb.rs +++ b/sdsearch-core/tests/zsl_query_parity_kb.rs @@ -7,22 +7,33 @@ use std::collections::HashMap; use std::path::PathBuf; #[derive(serde::Deserialize)] -struct Expected { queries: HashMap> } +struct Expected { + queries: HashMap>, +} #[derive(serde::Deserialize)] -struct Hit { id: usize } +struct Hit { + id: usize, +} fn expected() -> Expected { let raw = std::fs::read_to_string(concat!( - env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_expected_kb.json" - )).expect("kb oracle missing — run the KB generator"); + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_expected_kb.json" + )) + .expect("kb oracle missing — run the KB generator"); serde_json::from_str(&raw).unwrap() } fn seg() -> ZslSegment { ZslSegment::open(&PathBuf::from(concat!( - env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_kb" - ))).unwrap() + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_kb" + ))) + .unwrap() +} +fn sorted(mut v: Vec) -> Vec { + v.sort(); + v } -fn sorted(mut v: Vec) -> Vec { v.sort(); v } fn engine_set(hits: &[sdsearch_core::search::Hit]) -> Vec { sorted(hits.iter().map(|h| h.id).collect()) } @@ -33,32 +44,59 @@ fn oracle_set(e: &Expected, k: &str) -> Vec { #[test] fn term_queries_match_oracle_and_diverge() { let (s, e) = (seg(), expected()); - assert_eq!(engine_set(&term_query(&s, "title", "vpn", 0.0, 100)), oracle_set(&e, "term:title:vpn")); - assert_eq!(engine_set(&term_query(&s, "title", "laptop", 0.0, 100)), oracle_set(&e, "term:title:laptop")); + assert_eq!( + engine_set(&term_query(&s, "title", "vpn", 0.0, 100)), + oracle_set(&e, "term:title:vpn") + ); + assert_eq!( + engine_set(&term_query(&s, "title", "laptop", 0.0, 100)), + oracle_set(&e, "term:title:laptop") + ); // real divergence check: vpn and laptop match different docs - assert_ne!(oracle_set(&e, "term:title:vpn"), oracle_set(&e, "term:title:laptop")); + assert_ne!( + oracle_set(&e, "term:title:vpn"), + oracle_set(&e, "term:title:laptop") + ); } #[test] fn wildcard_query_matches_oracle() { let (s, e) = (seg(), expected()); // print* expands over several title terms (print, printer, printing, …) - assert_eq!(engine_set(&wildcard_query(&s, "title", "print*", 0, 0.0, 100)), oracle_set(&e, "wildcard:title:print*")); - assert!(oracle_set(&e, "wildcard:title:print*").len() >= 2, "wildcard should expand to multiple docs"); + assert_eq!( + engine_set(&wildcard_query(&s, "title", "print*", 0, 0.0, 100)), + oracle_set(&e, "wildcard:title:print*") + ); + assert!( + oracle_set(&e, "wildcard:title:print*").len() >= 2, + "wildcard should expand to multiple docs" + ); } #[test] fn fuzzy_typo_matches_oracle() { let (s, e) = (seg(), expected()); // "passwrd" (typo of "password"), similarity 0.5 / prefix 3 - assert_eq!(engine_set(&fuzzy_query(&s, "title", "passwrd", 0.5, 3, 0.0, 100)), oracle_set(&e, "fuzzy:title:passwrd:0.5:3")); + assert_eq!( + engine_set(&fuzzy_query(&s, "title", "passwrd", 0.5, 3, 0.0, 100)), + oracle_set(&e, "fuzzy:title:passwrd:0.5:3") + ); } #[test] fn phrase_queries_match_oracle() { let (s, e) = (seg(), expected()); - assert_eq!(engine_set(&phrase_query(&s, "title", &["setting", "up"], 0.0, 100)), oracle_set(&e, "phrase:title:setting up")); - assert_eq!(engine_set(&phrase_query(&s, "title", &["cloud", "storage"], 0.0, 100)), oracle_set(&e, "phrase:title:cloud storage")); + assert_eq!( + engine_set(&phrase_query(&s, "title", &["setting", "up"], 0.0, 100)), + oracle_set(&e, "phrase:title:setting up") + ); + assert_eq!( + engine_set(&phrase_query(&s, "title", &["cloud", "storage"], 0.0, 100)), + oracle_set(&e, "phrase:title:cloud storage") + ); // "setting up" matches 2 docs, "cloud storage" 1 → positions/adjacency genuinely exercised - assert_ne!(oracle_set(&e, "phrase:title:setting up"), oracle_set(&e, "phrase:title:cloud storage")); + assert_ne!( + oracle_set(&e, "phrase:title:setting up"), + oracle_set(&e, "phrase:title:cloud storage") + ); } diff --git a/sdsearch-core/tests/zsl_reader_during_write.rs b/sdsearch-core/tests/zsl_reader_during_write.rs index a0cf20d..d5a4789 100644 --- a/sdsearch-core/tests/zsl_reader_during_write.rs +++ b/sdsearch-core/tests/zsl_reader_during_write.rs @@ -13,11 +13,16 @@ fn temp_kb() -> PathBuf { let n = COUNTER.fetch_add(1, Ordering::Relaxed); let dir = std::env::temp_dir().join(format!("sdsearch_rdwr_{}_{}", std::process::id(), n)); std::fs::create_dir_all(&dir).unwrap(); - let src = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_kb")); + let src = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_kb" + )); for f in std::fs::read_dir(&src).unwrap() { let f = f.unwrap().path(); let name = f.file_name().unwrap().to_string_lossy(); - if name.contains("lock") || name.ends_with(".sti") { continue; } + if name.contains("lock") || name.ends_with(".sti") { + continue; + } std::fs::copy(&f, dir.join(f.file_name().unwrap())).unwrap(); } dir diff --git a/sdsearch-core/tests/zsl_unclean_shutdown.rs b/sdsearch-core/tests/zsl_unclean_shutdown.rs index f419b65..54c78a5 100644 --- a/sdsearch-core/tests/zsl_unclean_shutdown.rs +++ b/sdsearch-core/tests/zsl_unclean_shutdown.rs @@ -4,8 +4,8 @@ use sdsearch_core::index::IndexReader; use sdsearch_core::zsl::index::ZslIndex; -use sdsearch_core::zsl::writer::{IndexWriter, WriterOpts}; use sdsearch_core::zsl::segments::read_segment_infos; +use sdsearch_core::zsl::writer::{IndexWriter, WriterOpts}; use std::path::PathBuf; use std::sync::atomic::{AtomicU32, Ordering}; @@ -15,11 +15,16 @@ fn temp_kb() -> PathBuf { let n = COUNTER.fetch_add(1, Ordering::Relaxed); let dir = std::env::temp_dir().join(format!("sdsearch_unclean_{}_{}", std::process::id(), n)); std::fs::create_dir_all(&dir).unwrap(); - let src = PathBuf::from(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/zsl_index_kb")); + let src = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/zsl_index_kb" + )); for f in std::fs::read_dir(&src).unwrap() { let f = f.unwrap().path(); let name = f.file_name().unwrap().to_string_lossy(); - if name.contains("lock") || name.ends_with(".sti") { continue; } + if name.contains("lock") || name.ends_with(".sti") { + continue; + } std::fs::copy(&f, dir.join(f.file_name().unwrap())).unwrap(); } dir @@ -32,22 +37,31 @@ fn opens_clean_with_orphan_cfs_and_stale_lock_file() { // simulate a crash: an orphan .cfs NOT listed in segments_N + a stale write.lock.file. // the orphan's bytes = a copy of an existing valid .cfs under an unreferenced name. - let existing_cfs = std::fs::read_dir(&dir).unwrap() + let existing_cfs = std::fs::read_dir(&dir) + .unwrap() .filter_map(|e| e.ok().map(|e| e.path())) .find(|p| p.extension().map(|x| x == "cfs").unwrap_or(false)) .expect("the KB fixture has a .cfs"); std::fs::copy(&existing_cfs, dir.join("_99.cfs")).unwrap(); // orphan (not in segments_N) - std::fs::write(dir.join("write.lock.file"), b"").unwrap(); // stale lock file (not held) + std::fs::write(dir.join("write.lock.file"), b"").unwrap(); // stale lock file (not held) // the reader ignores the orphan and sees exactly the base docs. let reader = ZslIndex::open(&dir).unwrap(); - assert_eq!(reader.num_docs(), base, "the orphan _99.cfs must not be counted"); + assert_eq!( + reader.num_docs(), + base, + "the orphan _99.cfs must not be counted" + ); // segments_N does not list the orphan. let infos = read_segment_infos(&dir).unwrap(); - assert!(!infos.iter().any(|s| s.name == "_99"), "_99 should not be in segments_N"); + assert!( + !infos.iter().any(|s| s.name == "_99"), + "_99 should not be in segments_N" + ); // the writer opens: the stale lock file is NOT held → acquire OK. - let w = IndexWriter::open(&dir, WriterOpts::default()).expect("writer must open despite the leftover lock file"); + let w = IndexWriter::open(&dir, WriterOpts::default()) + .expect("writer must open despite the leftover lock file"); drop(w); std::fs::remove_dir_all(&dir).ok(); diff --git a/sdsearch-php/src/lib.rs b/sdsearch-php/src/lib.rs index bbc8a38..0b6f7da 100644 --- a/sdsearch-php/src/lib.rs +++ b/sdsearch-php/src/lib.rs @@ -64,22 +64,38 @@ fn run(index_dir: &str, params_json: &str) -> Result { where_groups: dto .r#where .into_iter() - .map(|w| WhereGroup { field: w.field, values: w.values, occur: occur_from(&w.occur) }) + .map(|w| WhereGroup { + field: w.field, + values: w.values, + occur: occur_from(&w.occur), + }) .collect(), in_groups: dto .r#in .into_iter() - .map(|i| InGroup { field: i.field, values: i.values }) + .map(|i| InGroup { + field: i.field, + values: i.values, + }) .collect(), fuzzy_similarity: 0.5, fuzzy_prefix_len: 3, wildcard_min_prefix: 0, }; - let hits = search_index(Path::new(index_dir), ¶ms, dto.min_score, dto.limit as usize) - .map_err(|e| format!("sdsearch: {e}"))?; + let hits = search_index( + Path::new(index_dir), + ¶ms, + dto.min_score, + dto.limit as usize, + ) + .map_err(|e| format!("sdsearch: {e}"))?; let out: Vec = hits .into_iter() - .map(|h| HitDto { id: h.id as u64, score: h.score, fields: h.fields }) + .map(|h| HitDto { + id: h.id as u64, + score: h.score, + fields: h.fields, + }) .collect(); serde_json::to_string(&out).map_err(|e| format!("sdsearch: serialize hits: {e}")) } @@ -180,7 +196,10 @@ fn resolve_doc_id(index: &ZslIndex, id_field: &str, value: &str) -> Result Result Vec { - let query = Query::Term { field: Some(field.to_string()), text: value.to_string() }; + let query = Query::Term { + field: Some(field.to_string()), + text: value.to_string(), + }; let hits = search(index, &query, 0.0, usize::MAX); hits.iter().map(|h| h.id as i64).collect() } @@ -205,7 +227,10 @@ fn resolve_doc_ids(index: &ZslIndex, field: &str, value: &str) -> Vec { #[php(change_method_case = "none")] impl Writer { pub fn __construct() -> Self { - Writer { inner: None, reader: None } + Writer { + inner: None, + reader: None, + } } /// opens the streaming writer over an existing ZSL index (takes the write-lock) + a cached @@ -252,7 +277,9 @@ impl Writer { } Ok(Ok(None)) => Ok(false), Ok(Err(msg)) => Err(PhpException::default(msg)), - Err(_) => Err(PhpException::default("sdsearch: panic in try_open".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in try_open".to_string(), + )), } } @@ -264,11 +291,15 @@ impl Writer { .reader .as_ref() .ok_or_else(|| PhpException::default("sdsearch: writer not open".to_string()))?; - let result = catch_unwind(AssertUnwindSafe(|| resolve_doc_id(index, &id_field, &value))); + let result = catch_unwind(AssertUnwindSafe(|| { + resolve_doc_id(index, &id_field, &value) + })); match result { Ok(Ok(v)) => Ok(v), Ok(Err(msg)) => Err(PhpException::default(msg)), - Err(_) => Err(PhpException::default("sdsearch: panic in find_doc_id".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in find_doc_id".to_string(), + )), } } @@ -285,7 +316,9 @@ impl Writer { let result = catch_unwind(AssertUnwindSafe(|| resolve_doc_ids(index, &field, &value))); match result { Ok(v) => Ok(v), - Err(_) => Err(PhpException::default("sdsearch: panic in find_doc_ids".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in find_doc_ids".to_string(), + )), } } @@ -305,7 +338,9 @@ impl Writer { })); match result { Ok(()) => Ok(()), - Err(_) => Err(PhpException::default("sdsearch: panic in delete_document".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in delete_document".to_string(), + )), } } @@ -317,12 +352,15 @@ impl Writer { .ok_or_else(|| PhpException::default("sdsearch: writer not open".to_string()))?; let result = catch_unwind(AssertUnwindSafe(|| { let doc = doc_from_json(&doc_json)?; - iw.add_document(doc).map_err(|e| format!("sdsearch: add: {e}")) + iw.add_document(doc) + .map_err(|e| format!("sdsearch: add: {e}")) })); match result { Ok(Ok(())) => Ok(()), Ok(Err(msg)) => Err(PhpException::default(msg)), - Err(_) => Err(PhpException::default("sdsearch: panic in add_document".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in add_document".to_string(), + )), } } @@ -339,7 +377,9 @@ impl Writer { match result { Ok(Ok(report)) => Ok(report.doc_count as i64), Ok(Err(msg)) => Err(PhpException::default(msg)), - Err(_) => Err(PhpException::default("sdsearch: panic in commit".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in commit".to_string(), + )), } } @@ -351,12 +391,15 @@ impl Writer { .ok_or_else(|| PhpException::default("sdsearch: writer not open".to_string()))?; self.reader = None; let result = catch_unwind(AssertUnwindSafe(|| { - iw.optimize().map_err(|e| format!("sdsearch: optimize: {e}")) + iw.optimize() + .map_err(|e| format!("sdsearch: optimize: {e}")) })); match result { Ok(Ok(report)) => Ok(report.doc_count as i64), Ok(Err(msg)) => Err(PhpException::default(msg)), - Err(_) => Err(PhpException::default("sdsearch: panic in optimize".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in optimize".to_string(), + )), } } @@ -369,7 +412,9 @@ impl Writer { let result = catch_unwind(AssertUnwindSafe(|| iw.document_count())); match result { Ok(count) => Ok(count as i64), - Err(_) => Err(PhpException::default("sdsearch: panic in document_count".to_string())), + Err(_) => Err(PhpException::default( + "sdsearch: panic in document_count".to_string(), + )), } } }