From 6ee76c9c244d4877275e821f5dac05521c5aadef Mon Sep 17 00:00:00 2001 From: Federico Martinez Date: Fri, 10 Jul 2026 20:10:15 -0300 Subject: [PATCH 1/5] build: tune release profile (fat LTO + strip symbols) Add [profile.release] to the workspace root: strip="symbols", lto="fat", codegen-units=1. Shrinks the PHP extension from 3.68 MB to 2.36 MB (-36%). Deliberately keeps panic="unwind" (the default): the sdsearch-php FFI boundary relies on catch_unwind to turn a Rust panic into a catchable PhpException and keep the PHP worker alive; panic="abort" would break that. Documented as an invariant in the manifest. --- Cargo.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 8c801c9..a4ddef3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,18 @@ [workspace] resolver = "2" members = ["sdsearch-core", "sdsearch-php"] + +# Release profile for the shipped binaries (also used by CI's --release builds, so CI +# smoke-tests exactly what we ship). +# +# INVARIANT: do NOT set `panic = "abort"`. The PHP FFI boundary in sdsearch-php wraps every +# call in catch_unwind to turn a Rust panic into a catchable PhpException and keep the PHP +# worker alive. `panic = "abort"` would make catch_unwind catch nothing and abort the whole +# process on any panic. It must stay the default (`unwind`). +# +# Also do NOT build with `-C target-cpu=native` for distributed binaries: it bakes the build +# runner's CPU features in and can SIGILL on older deployment CPUs. Keep the x86-64 baseline. +[profile.release] +strip = "symbols" # drop the symbol table: smaller binary, no internal-name leakage +lto = "fat" # whole-program LTO: maximum optimization (build time is not a concern) +codegen-units = 1 # let the optimizer work across the whole crate From 718cee541d7ca2fa9cf39e17ebe0ed3ac778e08e Mon Sep 17 00:00:00 2001 From: Federico Martinez Date: Fri, 10 Jul 2026 20:10:15 -0300 Subject: [PATCH 2/5] ci: run push builds only on main + tags, add concurrency on.push had no branch filter, so every job ran on both push and pull_request: a push to a branch with an open PR built the whole pipeline twice. Restrict push to main and v* tags (feature branches are covered by their pull_request run) and add a concurrency group that cancels superseded in-flight runs. --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b764c..8324524 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,20 @@ name: ci # Gates (test, lint) run first and BLOCK the build jobs (needs:). The coverage job is # informational (continue-on-error, in no job's needs) so a coverage/token hiccup never # reddens the pipeline. Windows uses nightly Rust (ext-php-rs abi_vectorcall). +# push runs only on main (merge) and release tags; feature-branch pushes are covered by +# their pull_request run, so a branch with an open PR no longer builds the whole pipeline +# twice. `concurrency` cancels a superseded run when you push again over an in-flight one. on: push: + branches: [main] + tags: ['v*'] pull_request: workflow_dispatch: {} +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read From 40b7c3da687d8f0169496ea48b2c530ce74924b2 Mon Sep 17 00:00:00 2001 From: Federico Martinez Date: Fri, 10 Jul 2026 20:12:29 -0300 Subject: [PATCH 3/5] docs: add byte-level ZSL format reference (docs/FORMAT.md) Byte-by-byte layout of every ZSL index file (.cfs, .fnm, .fdt/.fdx, .tis/.tii, .frq/.prx, .nrm, .del, segments.gen/segments_N) with ASCII field diagrams, the modified-UTF-8 / SmallFloat / VInt quirks, and worked hex examples. Derived from the parsers in sdsearch-core/src/zsl. Linked from ARCHITECTURE.md and the README. --- ARCHITECTURE.md | 4 + README.md | 3 +- docs/FORMAT.md | 371 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 docs/FORMAT.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 61c16cd..3434376 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -22,6 +22,10 @@ The crate is split in two: ## Format modules (`sdsearch-core/src/zsl/`) +> For the byte-level layout of each file type (field offsets, encodings, worked hex +> examples), see [`docs/FORMAT.md`](docs/FORMAT.md). This section maps those file types to +> the modules that parse and serialize them. + The reader is organized as one module per piece of the ZSL on-disk format, each a self-contained parser for that file type: diff --git a/README.md b/README.md index 1da673c..fb1ef72 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ feature set. crashed worker). See [`ARCHITECTURE.md`](ARCHITECTURE.md) for the module layout, the reader/writer design, -and the byte-fidelity details that make the format compatibility work. +and the byte-fidelity details that make the format compatibility work, and +[`docs/FORMAT.md`](docs/FORMAT.md) for the byte-level layout of every ZSL index file. ## Building diff --git a/docs/FORMAT.md b/docs/FORMAT.md new file mode 100644 index 0000000..9a33d8d --- /dev/null +++ b/docs/FORMAT.md @@ -0,0 +1,371 @@ +# ZSL on-disk index format + +This is a byte-level reference for the [Zend Search Lucene](https://framework.zend.com/manual/1.12/en/zend.search.lucene.html) +(ZSL) index format as read and written by `sdsearch`. It complements +[`ARCHITECTURE.md`](../ARCHITECTURE.md), which describes the module layout; here we +describe the bytes themselves. + +The layouts below are derived directly from the parsers in `sdsearch-core/src/zsl/` +(readers) and `sdsearch-core/src/zsl/writer/` (serializers). The goal of `sdsearch` is +byte-for-byte compatibility with a real ZSL install in both directions, so this document +describes the *actual* wire format, not an idealized one. + +> Scope note: `sdsearch` supports the subset of the format an optimized ZSL index uses in +> practice. Two variants are intentionally **not** supported: the sparse/DGaps `.del` +> layout, and per-field "separate norm files" (an unoptimized index). Both surface as an +> error at open time rather than silent misreads. They are called out in the relevant +> sections below. + +--- + +## 1. Primitive types + +Every file is built from five primitives. All fixed-width integers are **big-endian**. + +| Type | Size | Encoding | +|---|---|---| +| `Byte` | 1 | raw byte | +| `Int32` | 4 | big-endian, two's-complement when signed | +| `Int64` | 8 | big-endian | +| `VInt` | 1–10 | unsigned LEB128: 7 payload bits per byte, low group first, high bit (`0x80`) = "more bytes follow" | +| `String` | var | `VInt` character count, then the characters in *modified UTF-8* (see §4) | + +`VInt` example: `300` = `0b1_0010_1100` → low 7 bits `0101100` with continuation, then +`0000010` → bytes `AC 02`. + +Source: `sdsearch-core/src/serialize.rs` (`read_vint`/`write_vint`) and +`sdsearch-core/src/zsl/bytes.rs` (integers + strings). + +--- + +## 2. Directory anatomy + +An index is a directory. A generation pointer names the current segment list; the segment +list names the live segments; each segment is a single compound file (`.cfs`) that packs +all of that segment's sub-files, except the deletions bitmap (`.del`), which lives beside +the `.cfs` so it can be rewritten without touching the segment. + +``` +/ +├── segments.gen generation pointer → which segments_N is current +├── segments_ the live segment list at generation N +├── _.cfs segment "_": compound file (see §3) +├── _.del (or __.del) deletions for "_" (outside the .cfs) +├── _.cfs segment "_" +└── ... + +_.cfs ─packs→ _.fnm field infos (§5) + _.fdt stored field data (§6) + _.fdx stored field index (§6) + _.tis term dictionary (§7) + _.tii term index (sparse) (§7) + _.frq postings: doc + freq (§8) + _.prx postings: positions (§8) + _.nrm length norms (§9) +``` + +**Multi-segment model.** Incremental writes append new segments; `optimize()`/merge +collapses them into one. A logical index is the concatenation of its segments in list +order: `zsl/index.rs` gives each segment a global document-ID base equal to the running +sum of prior segments' `docCount` (deletes included — the same numbering Lucene's +multi-segment readers use) and routes each lookup to the owning segment. + +**Visibility / durability.** A freshly written segment file exists on disk but is +**invisible** until a new `segments_N` lists it and `segments.gen` points at that +generation. The generation flip is the atomic commit point: if the process dies mid-batch, +the old generation is still intact and the orphan segment files are simply never +referenced. + +--- + +## 3. `.cfs` — compound file + +A directory of named sub-files followed by their concatenated bodies. + +``` +┌────────┬─────────────────────────────┐ +│ VInt │ entryCount │ +├────────┼──────────────┬──────────────┤ +│ Int64 │ offset │ ┐ │ +│ String │ name │ ├─ × entryCount (the directory) +├────────┴──────────────┴──────────────┤ +│ │ sub-file i = bytes [offset_i, offset_{i+1}) +└───────────────────────────────────────┘ last sub-file runs to EOF +``` + +`offset` is an absolute byte offset into the `.cfs` where that sub-file's body begins. +The reader mmaps the whole file and hands out `[offset_i, offset_{i+1})` slices (the last +runs to EOF), validating that offsets are non-decreasing and in range. + +Source: `zsl/cfs.rs`, `zsl/writer/cfs.rs`. + +--- + +## 4. Modified UTF-8 strings + +A `String` is `VInt(charCount)` followed by the characters. Two things differ from plain +byte-length-prefixed UTF-8 and silently corrupt cross-engine reads if missed: + +- **The prefix counts *characters* (code points), not bytes.** A 3-code-point string of + multibyte characters has prefix `3` but more than 3 bytes of body. +- **NUL (`U+0000`) is encoded as two bytes `C0 80`**, never as `00` (Java-style modified + UTF-8). Every other code point uses standard UTF-8, **including 4-byte sequences** for + the supplementary planes (emoji, CJK extensions). ZSL's PHP writer stores a supplementary + code point as one standard 4-byte sequence counted as one character — not as a UTF-16 + surrogate pair — and the reader decodes it the same way, so round-trips are exact. + +Source: `zsl/bytes.rs` (`read_modified_utf8`/`write_modified_utf8`). The 4-byte branch +matters for real data: documents with emoji drifted before it was handled. + +--- + +## 5. `.fnm` — field infos + +Field number → name + flags. A field's number is its ordinal in this list (0-based); every +other file refers to fields by that number. + +``` +┌────────┬──────────────┐ +│ VInt │ fieldCount │ +├────────┼──────────────┤ +│ String │ name │ ┐ +│ Byte │ flags │ ├─ × fieldCount +└────────┴──────────────┘ ┘ +flags: bit0 = indexed, bit1 = tokenized, bit2 = stores norms, ... +``` + +`sdsearch`'s reader keeps `name` and the `indexed` bit (bit 0); the other bits exist in the +byte and are preserved by the writer. + +Source: `zsl/fields.rs`. + +--- + +## 6. `.fdt` / `.fdx` — stored fields + +Stored fields are the verbatim values returned with a hit (as opposed to the inverted index +used to match). `.fdx` is a flat offset table; `.fdt` holds the data. + +**`.fdx` (index):** one `Int64` per document — the byte offset of that document's record in +`.fdt`. The document count of the segment is `len(.fdx) / 8`. + +``` +.fdx: [ Int64 offset_doc0 ][ Int64 offset_doc1 ]... (docCount entries) +``` + +**`.fdt` (data):** at a document's offset, + +``` +┌────────┬──────────────┐ +│ VInt │ storedCount │ +├────────┼──────────────┤ +│ VInt │ fieldNum │ ┐ fieldNum is LOCAL to the segment +│ Byte │ flags │ │ flags: bit0 = tokenized, bit1 = binary +│ value │ … │ ├─ × storedCount +└────────┴──────────────┘ ┘ + value = String (modified UTF-8) when bit1 (binary) = 0 + = VInt byteLen + byteLen bytes when bit1 (binary) = 1 +``` + +The host application stores only text, so `binary` is expected to be 0; the reader handles +the binary branch defensively. Tokenized `Text` fields carry a trailing `\n` that ZSL's +`compactText()` adds — it is a faithful part of the stored bytes and is **not** trimmed. + +Source: `zsl/stored.rs`. + +--- + +## 7. `.tis` / `.tii` — term dictionary + +**`.tis` (full dictionary)** — all terms, grouped by field and sorted by text within a +field, with each term's `docFreq` and pointers into `.frq`/`.prx`. + +``` +┌────────┬──────────────────┐ +│ Int32 │ marker 0xFFFFFFFD │ FORMAT_2_1 +│ Int64 │ termCount │ +│ Int32 │ indexInterval │ +│ Int32 │ skipInterval │ +│ Int32 │ maxSkipLevels │ +├────────┼──────────────────┤ +│ VInt │ sharedPrefixChars │ ┐ shared with the PREVIOUS term's text (see below) +│ String │ suffix │ │ +│ VInt │ fieldNum │ ├─ × termCount +│ VInt │ docFreq │ │ +│ VInt │ freqDelta │ │ accumulates → absolute .frq pointer +│ VInt │ proxDelta │ │ accumulates → absolute .prx pointer +└────────┴──────────────────┘ ┘ +``` + +Prefix sharing: a term's text is `previousText[0..sharedPrefixChars] + suffix`. The writer +emits `sharedPrefixChars = 0` whenever the field number changes, so applying the shared +prefix against the previous text (regardless of field) reproduces the dictionary exactly. +The `freqDelta`/`proxDelta` are cumulative across the **whole file**, not reset per field. +`skipOffset` is omitted because skips are disabled (`docFreq < skipInterval`). + +**`.tii` (sparse index)** — the same entry shape plus an `IndexDelta` (`VInt`) pointing into +`.tis`, holding every `indexInterval`-th term so a reader can seek without scanning the +whole `.tis`. `sdsearch`'s reader **does not read `.tii`**: it loads the entire `.tis` into +a compact in-memory buffer and binary-searches it. The writer still emits `.tii` (with a +header and a synthetic initial entry) because a stock ZSL install requires it. + +Source: `zsl/terms.rs` (reader), `zsl/writer/terms.rs` (both files). + +--- + +## 8. `.frq` / `.prx` — postings + +Per-term document and position lists, reached via the term's `freqPointer`/`proxPointer` +from `.tis`. + +**`.frq` (docs + freqs):** `docFreq` entries, doc IDs delta-encoded and ascending. The +term-frequency-of-1 case is folded into the doc delta's low bit: + +``` +per doc (× docFreq): + VInt v + docDelta = v >> 1 docId = previousDocId + docDelta + if (v & 1) == 1: freq = 1 (implicit, no extra byte) + else: VInt freq +``` + +**`.prx` (positions):** for each doc, `freq` position deltas (ascending, cumulative): + +``` +per doc (× docFreq): ← walk .frq in parallel to know `freq` per doc + per occurrence (× freq): + VInt posDelta position = previousPosition + posDelta +``` + +Source: `zsl/postings.rs`. + +--- + +## 9. `.nrm` — length norms + +One norm byte per document per **indexed** field, used to weight shorter fields higher at +score time. + +``` +┌──────────────┬───────────────────────────────┐ +│ 3 bytes │ 'N' 'R' 'M' │ header +│ Byte │ format │ +├──────────────┼───────────────────────────────┤ +│ Byte × docCount │ norms for indexed field #1 │ fields in field-number order +│ Byte × docCount │ norms for indexed field #2 │ +│ ... │ ... │ +└──────────────┴───────────────────────────────┘ +``` + +Each norm byte is Lucene's `SmallFloat` (a.k.a. `Similarity::decodeNorm`) byte-float: + +``` +if byte == 0: norm = 0.0 +mantissa = byte & 0x07 +exponent = (byte >> 3) & 0x1F +norm = f32::from_bits( (exponent << 24) | (mantissa << 21) ) +approx field length ≈ 1 / norm² (inverse of encodeNorm(1/√len)) +``` + +Source: `zsl/norms.rs`. + +--- + +## 10. `.del` — deletions + +A bitset of deleted document IDs for one segment; a set bit means the local document is +deleted. It lives **outside** the `.cfs` so deletions can be rewritten without touching the +segment. The delete generation in `segments_N` selects the filename: `-1` = none, `0` = +`.del`, `>0` = `_.del`. + +Only the **dense** BitVector layout (pre-2.1) is supported: + +``` +┌────────┬──────────────┐ +│ Int32 │ docCount │ (if this reads 0xFFFFFFFF it's the SPARSE layout → error) +│ Int32 │ bitCount │ +├────────┼──────────────┤ +│ Byte × │ bitmap │ doc d deleted ⟺ bit (d % 8) of byte (d / 8) is set +└────────┴──────────────┘ +``` + +The sparse/DGaps layout (first `Int32` == `0xFFFFFFFF`, then a `(VInt dgap, Byte)` stream) +is rejected with an error rather than misread. + +Source: `zsl/deletes.rs`. + +--- + +## 11. `segments.gen` and `segments_N` + +**`segments.gen`** — the generation pointer, written twice for torn-write detection. + +``` +┌────────┬──────────────┐ +│ Int32 │ format 0xFFFFFFFE │ +│ Int64 │ gen1 │ +│ Int64 │ gen2 │ must equal gen1 (else the file was caught mid-write) +└────────┴──────────────┘ +``` + +The current segment list is `segments_`, or `segments` when `gen == 0`. +`base36` is lowercase (`base_convert(n, 10, 36)` in ZSL): `10 → "a"`, `46 → "1a"`. + +**`segments_N`** — the live segment list. + +``` +┌────────┬───────────────────────────────┐ +│ Int32 │ format 0xFFFFFFFC (2.3) | 0xFFFFFFFD (2.1) │ +│ Int64 │ version │ +│ Int32 │ nameCounter │ +│ Int32 │ segCount │ +├────────┼───────────────────────────────┤ per segment (× segCount): +│ String │ name │ +│ Int32 │ docCount (maxDoc, incl. deletes) │ +│ Int64 │ delGen (-1 none / 0 / >0) │ +│ ─ if format 2.3: ─────────────────────│ +│ Int32 │ docStoreOffset │ if != 0xFFFFFFFF: +│ String │ docStoreSegment (optional) │ String + Byte follow +│ Byte │ docStoreIsCompound (optional)│ +│ ───────────────────────────────────────│ +│ Byte │ hasSingleNormFile │ +│ Int32 │ numField (must be 0xFFFFFFFF)│ anything else = separate norm files → error +│ Byte │ isCompound │ +└────────┴───────────────────────────────┘ +``` + +`docCount` is the per-segment maxDoc (deletes included) and is what the global document-ID +base is built from. `numField` other than `0xFFFFFFFF` means "separate norm files" (an +unoptimized index), which neither ZSL nor `sdsearch` supports — optimize the index first. + +Source: `zsl/segments.rs`. + +--- + +## 12. Worked hex examples + +**`.fnm` with two fields** — `title` (indexed) and `id_attr` (not indexed): + +``` +02 VInt fieldCount = 2 +05 74 69 74 6C 65 01 String len=5 "title", flags=0x01 (indexed) +07 69 64 5F 61 74 74 72 00 String len=7 "id_attr", flags=0x00 (not indexed) +``` + +**`.del` with 10 docs, only doc 2 deleted:** + +``` +00 00 00 0A Int32 docCount = 10 +00 00 00 01 Int32 bitCount = 1 +04 bitmap: 0b0000_0100 → bit 2 set → doc 2 deleted +``` + +**`String "hi"`** and a **NUL character**: + +``` +02 68 69 "hi": VInt charCount=2, 'h' 'i' +01 C0 80 "\0": VInt charCount=1, NUL encoded C0 80 +``` + +These match the round-trip assertions in `zsl/bytes.rs`, `zsl/fields.rs`, and +`zsl/deletes.rs`. From 1ee902c148308ce7486be75183d5d75b80ec81c2 Mon Sep 17 00:00:00 2001 From: Federico Martinez Date: Fri, 10 Jul 2026 20:15:13 -0300 Subject: [PATCH 4/5] docs: document the PHP API (stub + docs/API.md) Add sdsearch.stub.php: signatures + PHPDoc for the whole extension surface (sdsearch_version, SdSearch\Engine, SdSearch\Writer) for IDEs/PHPStan; never loaded at runtime. Add docs/API.md: narrative guide with end-to-end indexing/update/search examples, the params + hit JSON contracts, and the field kinds. Symbol names verified by Reflection against the compiled extension; stub passes 'php -n -l'. Linked from README. --- README.md | 4 +- docs/API.md | 162 +++++++++++++++++++++++++++++++++++++++++ sdsearch.stub.php | 181 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 docs/API.md create mode 100644 sdsearch.stub.php diff --git a/README.md b/README.md index fb1ef72..4fc5c6d 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,9 @@ feature set. See [`ARCHITECTURE.md`](ARCHITECTURE.md) for the module layout, the reader/writer design, and the byte-fidelity details that make the format compatibility work, and -[`docs/FORMAT.md`](docs/FORMAT.md) for the byte-level layout of every ZSL index file. +[`docs/FORMAT.md`](docs/FORMAT.md) for the byte-level layout of every ZSL index file. The +PHP API is documented in [`docs/API.md`](docs/API.md) (usage examples) and +[`sdsearch.stub.php`](sdsearch.stub.php) (signatures + PHPDoc for IDEs / PHPStan). ## Building diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..a98f80f --- /dev/null +++ b/docs/API.md @@ -0,0 +1,162 @@ +# PHP API + +The `sdsearch` extension exposes three symbols to PHP: the function `sdsearch_version()` +and the classes `SdSearch\Engine` (search) and `SdSearch\Writer` (indexing). All data +crosses the boundary as JSON strings. + +The full signatures with PHPDoc live in [`sdsearch.stub.php`](../sdsearch.stub.php) at the +repo root — point your IDE / PHPStan at it for autocompletion and type-checking (it is a +stub, never loaded at runtime). This page is the narrative guide with runnable examples. + +> **Error handling.** Every method throws a catchable `\Exception` on any failure +> (malformed JSON, missing index, lock contention, internal error). The FFI boundary is +> panic-safe: an internal Rust panic becomes an `\Exception`, never a crashed PHP worker. +> Wrap calls in `try/catch`. + +## Loading the extension + +```ini +; php.ini +extension=sdsearch.so ; Linux +; extension=sdsearch.dll ; Windows +``` + +```php +echo sdsearch_version(); // "0.1.0" — also a smoke test that the extension loaded +``` + +## Method reference + +### `SdSearch\Engine` + +| Method | Purpose | Throws | +|---|---|---| +| `__construct()` | Create an engine. | — | +| `search(string $indexDir, string $paramsJson): string` | Run a query, return hits as JSON. | bad params JSON, missing index, engine error | + +### `SdSearch\Writer` + +| Method | Purpose | Throws | +|---|---|---| +| `__construct()` | Create a writer (not yet open). | — | +| `open(string $indexDir): void` | Take the write-lock + open. | index locked, open error | +| `try_open(string $indexDir): bool` | Like `open` but returns `false` if busy. | any error except "locked" | +| `find_doc_id(string $idField, string $value): int` | `_key:value` → global doc id, or `-1`. | writer not open | +| `find_doc_ids(string $field, string $value): int[]` | Literal `:value` → all matching doc ids. | writer not open | +| `delete_document(int $docId): void` | Mark a doc deleted (neg/out-of-range = no-op). | writer not open | +| `add_document(string $docJson): void` | Buffer a doc for the next commit. | bad JSON, unknown kind, writer closed | +| `commit(): int` | Flush adds+deletes, **consume** the writer. | writer not open | +| `optimize(): int` | Commit then merge into one segment, **consume**. | writer not open | +| `document_count(): int` | Live base + buffered − deletes. | writer not open | + +`commit()` and `optimize()` consume the writer: after either, the object is closed and any +further method throws "writer not open". Create a fresh `Writer` for the next batch. + +## Indexing (write path) + +```php +use SdSearch\Writer; + +$indexDir = '/var/lib/app/search-index'; + +$w = new Writer(); +$w->open($indexDir); // throws if another writer holds the lock + +// Add a document. Field kinds: "text" (tokenized+stored), "keyword" (exact+stored), +// "unindexed" (stored only). +$doc = [ + 'fields' => [ + ['name' => 'id_key', 'value' => '42', 'kind' => 'keyword'], + ['name' => 'title', 'value' => 'How to reset a password', 'kind' => 'text'], + ['name' => 'body', 'value' => 'Open settings, ...', 'kind' => 'text'], + ['name' => 'status', 'value' => 'published', 'kind' => 'keyword'], + ], +]; +$w->add_document(json_encode($doc)); + +$w->commit(); // flush; writer is now closed +``` + +### Updating an existing document + +Deletes are by internal doc id, so resolve first, then delete, then add the new version in +the same batch: + +```php +$w = new Writer(); +$w->open($indexDir); + +$docId = $w->find_doc_id('id', '42'); // resolves id_key:42 → global id, or -1 +if ($docId !== -1) { + $w->delete_document($docId); +} +$w->add_document(json_encode($updatedDoc)); + +$w->optimize(); // commit + compact into a single segment +``` + +### Non-blocking open for a background feed + +```php +$w = new Writer(); +if (!$w->try_open($indexDir)) { + // another worker is writing — skip this cycle instead of throwing/blocking + return; +} +// ... add/delete ... +$w->commit(); +``` + +## Searching (read path) + +```php +use SdSearch\Engine; + +$engine = new Engine(); + +$params = [ + 'text' => 'reset password', + 'where' => [ + ['field' => 'status', 'values' => ['published'], 'occur' => 'must'], + ], + 'in' => [ + ['field' => 'category_key', 'values' => ['10', '11']], + ], + 'min_score' => 0.0, + 'limit' => 20, +]; + +$json = $engine->search($indexDir, json_encode($params)); +$hits = json_decode($json, true); + +foreach ($hits as $hit) { + // $hit = ['id' => int, 'score' => float, 'fields' => ['name' => 'value', ...]] + printf("#%d score=%.3f %s\n", $hit['id'], $hit['score'], $hit['fields']['title'] ?? ''); +} +``` + +### Query parameters + +| Key | Type | Meaning | +|---|---|---| +| `text` | string | Free-text query over tokenized fields. | +| `where` | array | Each `{field, values[], occur}`; `occur` ∈ `must` \| `mustnot` \| `should` (default `should`). | +| `in` | array | Each `{field, values[]}`; matches the (literal, key-suffixed) field against any value. | +| `min_score` | float | Drop hits below this score. | +| `limit` | int | Maximum hits to return. | + +Each hit is `{ "id": int, "score": float, "fields": { name: value, ... } }`, where `id` is +the global internal document id and `fields` are the document's stored fields. + +## Wrapping it safely + +```php +try { + $json = (new Engine())->search($indexDir, json_encode($params)); + $hits = json_decode($json, true, flags: JSON_THROW_ON_ERROR); +} catch (\Exception $e) { + // missing index, malformed params, or internal engine error — never a crashed worker + error_log('sdsearch: ' . $e->getMessage()); + $hits = []; +} +``` diff --git a/sdsearch.stub.php b/sdsearch.stub.php new file mode 100644 index 0000000..03d0ed0 --- /dev/null +++ b/sdsearch.stub.php @@ -0,0 +1,181 @@ +_key:value` against the writer's base snapshot to a global + * internal document id. + * + * @param string $idField Logical id field name; the `_key` suffix is added internally. + * @param string $value The id value to look up. + * @return int The global internal document id, or `-1` if there is no match. + * @throws \Exception if the writer is not open, or on an internal error. + */ + public function find_doc_id(string $idField, string $value): int {} + + /** + * Resolves a LITERAL `:value` term query to ALL live matching global document + * ids. Unlike {@see Writer::find_doc_id()}, `$field` is used verbatim (already + * carrying its `_key` suffix as indexed). Use for multi-doc removal/dedup (e.g. a + * multi-language category whose id maps to several docs, or `language_key:`). + * + * @param string $field Literal indexed field name. + * @param string $value The value to match. + * @return int[] Global internal document ids (empty array if none match). + * @throws \Exception if the writer is not open, or on an internal error. + */ + public function find_doc_ids(string $field, string $value): array {} + + /** + * Marks a document deleted by its global internal id. A negative or out-of-range id + * is a silent no-op. Takes effect on the next + * {@see Writer::commit()}/{@see Writer::optimize()}. + * + * @param int $docId Global internal document id (from find_doc_id / find_doc_ids). + * @throws \Exception if the writer is not open, or on an internal error. + */ + public function delete_document(int $docId): void {} + + /** + * Buffers a document to be added on the next commit. + * + * `$docJson` is a JSON object: + * ```json + * { "fields": [ { "name": "title", "value": "Hello", "kind": "text" } ] } + * ``` + * `kind` is one of: + * - `"text"` — tokenized + indexed + stored (full-text searchable). + * - `"keyword"` — indexed as a single token + stored (exact-match / key fields). + * - `"unindexed"` — stored only, not searchable. + * + * @param string $docJson JSON-encoded document (see above). + * @throws \Exception on malformed JSON, an unknown `kind`, a closed writer, or an + * internal error. + */ + public function add_document(string $docJson): void {} + + /** + * Commits buffered additions and pending deletes, then **consumes** the writer + * (releases the lock and closes the index). + * + * @return int The document count the index will have after the commit. + * @throws \Exception if the writer is not open (e.g. already committed), or on error. + */ + public function commit(): int {} + + /** + * Commits, then merges all live segments (plus pending deletes) into a single + * compacted segment. **Consumes** the writer, like {@see Writer::commit()}. + * + * @return int The document count the index will have after the optimize. + * @throws \Exception if the writer is not open, or on error. + */ + public function optimize(): int {} + + /** + * Total documents the index will see after commit (live base + buffered − deletes). + * Requires an open writer. + * + * @return int + * @throws \Exception if the writer is not open, or on an internal error. + */ + public function document_count(): int {} + } +} From 0110ff3f2b1b413cc4db9f9164095cad183b6cb7 Mon Sep 17 00:00:00 2001 From: Federico Martinez Date: Fri, 10 Jul 2026 20:25:52 -0300 Subject: [PATCH 5/5] ci: add release workflow (Windows dll + Ubuntu/EL8 .so, PHP 8.4 NTS) On a v* tag, build + smoke-test three PHP 8.4 NTS binaries in parallel and publish them to a GitHub Release: - sdsearch-windows-x64-php84nts.dll (windows-latest, nightly + LLVM) - sdsearch-ubuntu-x64-php84nts.so (ubuntu-latest, glibc ~2.39) - sdsearch-el8-x64-php84nts.so (rockylinux:8 container, glibc 2.28) The EL8 build (Rocky 8 + Remi PHP 8.4 NTS) links only up to GLIBC_2.28, so it loads on RHEL/CentOS/Rocky/Alma 8 & 9 (verified locally in a rockylinux:8 container). ext-php-rs needs PHP/PHP_CONFIG set explicitly in the container. The publish job asserts the tag equals sdsearch-core's crate version, then attaches the binaries. workflow_dispatch runs a build-only dry run (no release). github.ref_name is passed via env and quoted. --- .github/workflows/release.yml | 170 ++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5656d31 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,170 @@ +name: release + +# Publishes the PHP 8.4 NTS extension binaries on a version tag (e.g. v0.1.0): +# - sdsearch-windows-x64-php84nts.dll (Windows) +# - sdsearch-ubuntu-x64-php84nts.so (glibc ~2.39: Ubuntu 22.04+/Debian recent) +# - sdsearch-el8-x64-php84nts.so (glibc 2.28: RHEL/CentOS/Rocky/Alma 8 & 9) +# The EL8 build links an older glibc, so it also loads on recent Ubuntu — it is the most +# portable Linux binary; the ubuntu build is shipped as well for parity with CI. +# +# workflow_dispatch does a dry run: it builds + smoke-tests + uploads artifacts but does NOT +# create a GitHub Release (only a tag push does). +on: + push: + tags: ['v*'] + workflow_dispatch: {} + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + build-windows: + name: Build + smoke (Windows, nightly, PHP 8.4 NTS) + runs-on: windows-latest + # ext-php-rs needs nightly on Windows for abi_vectorcall (RUSTUP_TOOLCHAIN overrides the + # stable pin in rust-toolchain.toml). + env: + RUSTUP_TOOLCHAIN: nightly + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + - uses: KyleMayes/install-llvm-action@v2 + with: + version: "18.1" + - run: cargo build -p sdsearch-php --release + - name: Smoke test (extension loads + sdsearch_version) + run: php -d extension="${{ github.workspace }}\target\release\sdsearch.dll" tests\smoke.php + - name: Stage artifact + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path dist | Out-Null + Copy-Item target\release\sdsearch.dll dist\sdsearch-windows-x64-php84nts.dll + - uses: actions/upload-artifact@v4 + with: + name: sdsearch-windows-x64-php84nts + path: dist/* + if-no-files-found: error + + build-ubuntu: + name: Build + smoke (Ubuntu, PHP 8.4 NTS) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: shivammathur/setup-php@v2 + with: + php-version: '8.4' + tools: none + extensions: none + - name: Assert PHP build is NTS + run: php -v | grep -q 'NTS' || { echo "::error::expected an NTS PHP build, got:"; php -v; exit 1; } + - name: Install clang (ext-php-rs bindgen) + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo build -p sdsearch-php --release + - name: Smoke test (extension loads + sdsearch_version) + run: php -d extension="$(pwd)/target/release/libsdsearch.so" tests/smoke.php + - name: Stage artifact + run: | + mkdir -p dist + cp target/release/libsdsearch.so dist/sdsearch-ubuntu-x64-php84nts.so + - uses: actions/upload-artifact@v4 + with: + name: sdsearch-ubuntu-x64-php84nts + path: dist/* + if-no-files-found: error + + build-el8: + name: Build + smoke (EL8, glibc 2.28, PHP 8.4 NTS) + runs-on: ubuntu-latest + # Build inside a Rocky 8 container (glibc 2.28) so the .so loads on RHEL/CentOS/Rocky/Alma + # 8 and 9. PHP 8.4 NTS dev headers come from the Remi repo. Rocky 8's glibc 2.28 is new + # enough for GitHub's bundled node20 actions (which is why CentOS 7 / glibc 2.17 is not an + # option here). + container: rockylinux:8 + # ext-php-rs's build script does not auto-detect PHP inside this container even though + # /usr/bin/php is on PATH; point it explicitly at the Remi PHP 8.4 binary + php-config. + env: + PHP: /usr/bin/php + PHP_CONFIG: /usr/bin/php-config + steps: + - name: Install PHP 8.4 NTS + build toolchain + run: | + set -euxo pipefail + dnf install -y epel-release + dnf install -y https://rpms.remirepo.net/enterprise/remi-release-8.rpm + dnf module reset -y php + dnf module enable -y php:remi-8.4 + dnf install -y php-cli php-devel clang llvm-devel gcc make git tar + - uses: actions/checkout@v4 + - name: Install Rust (stable) + run: | + set -euxo pipefail + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Assert PHP is 8.4 NTS + run: | + php -v + php -v | grep -q 'NTS' || { echo "::error::expected an NTS PHP build"; exit 1; } + php -v | grep -q '8\.4\.' || { echo "::error::expected PHP 8.4"; exit 1; } + - run: cargo build -p sdsearch-php --release + - name: Smoke test (extension loads + sdsearch_version) + run: php -d extension="$(pwd)/target/release/libsdsearch.so" tests/smoke.php + - name: Stage artifact + run: | + mkdir -p dist + cp target/release/libsdsearch.so dist/sdsearch-el8-x64-php84nts.so + - uses: actions/upload-artifact@v4 + with: + name: sdsearch-el8-x64-php84nts + path: dist/* + if-no-files-found: error + + publish: + name: Publish GitHub Release + runs-on: ubuntu-latest + needs: [build-windows, build-ubuntu, build-el8] + # Only publish on a tag push; workflow_dispatch stops after the builds (dry run). + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write # create the release + env: + TAG: ${{ github.ref_name }} + steps: + - uses: actions/checkout@v4 + - name: Assert tag matches crate version + run: | + set -euo pipefail + VERSION=$(grep -m1 '^version' sdsearch-core/Cargo.toml | sed -E 's/.*"(.*)".*/\1/') + echo "crate version: $VERSION, tag: $TAG" + if [ "v$VERSION" != "$TAG" ]; then + echo "::error::tag $TAG does not match crate version v$VERSION (bump sdsearch-core/Cargo.toml or fix the tag)" + exit 1 + fi + - name: Download all build artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + - name: List artifacts + run: ls -la artifacts + - name: Create release + uses: softprops/action-gh-release@v2 + with: + files: artifacts/* + fail_on_unmatched_files: true + generate_release_notes: true + body: | + PHP 8.4 NTS native extension binaries. + + | File | Platform | glibc | Loads on | + |---|---|---|---| + | `sdsearch-windows-x64-php84nts.dll` | Windows x64 | — | Windows, PHP 8.4 NTS | + | `sdsearch-ubuntu-x64-php84nts.so` | Linux x64 | ~2.39 | Ubuntu 22.04+/Debian recent, PHP 8.4 NTS | + | `sdsearch-el8-x64-php84nts.so` | Linux x64 | 2.28 | RHEL/CentOS/Rocky/Alma 8 & 9 (+ recent Ubuntu), PHP 8.4 NTS |