Better ci and docs

.github/workflows/ci.yml

@@ -3,11 +3,20 @@ name: ci
 # Gates (test, lint) run first and BLOCK the build jobs (needs:). The coverage job is
 # informational (continue-on-error, in no job's needs) so a coverage/token hiccup never
 # reddens the pipeline. Windows uses nightly Rust (ext-php-rs abi_vectorcall).
+# push runs only on main (merge) and release tags; feature-branch pushes are covered by
+# their pull_request run, so a branch with an open PR no longer builds the whole pipeline
+# twice. `concurrency` cancels a superseded run when you push again over an in-flight one.
 on:
   push:
+    branches: [main]
+    tags: ['v*']
   pull_request:
   workflow_dispatch: {}
 
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 

.github/workflows/release.yml

@@ -0,0 +1,170 @@
+name: release
+
+# Publishes the PHP 8.4 NTS extension binaries on a version tag (e.g. v0.1.0):
+#   - sdsearch-windows-x64-php84nts.dll   (Windows)
+#   - sdsearch-ubuntu-x64-php84nts.so     (glibc ~2.39: Ubuntu 22.04+/Debian recent)
+#   - sdsearch-el8-x64-php84nts.so        (glibc 2.28: RHEL/CentOS/Rocky/Alma 8 & 9)
+# The EL8 build links an older glibc, so it also loads on recent Ubuntu — it is the most
+# portable Linux binary; the ubuntu build is shipped as well for parity with CI.
+#
+# workflow_dispatch does a dry run: it builds + smoke-tests + uploads artifacts but does NOT
+# create a GitHub Release (only a tag push does).
+on:
+  push:
+    tags: ['v*']
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build-windows:
+    name: Build + smoke (Windows, nightly, PHP 8.4 NTS)
+    runs-on: windows-latest
+    # ext-php-rs needs nightly on Windows for abi_vectorcall (RUSTUP_TOOLCHAIN overrides the
+    # stable pin in rust-toolchain.toml).
+    env:
+      RUSTUP_TOOLCHAIN: nightly
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@nightly
+      - uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.4'
+      - uses: KyleMayes/install-llvm-action@v2
+        with:
+          version: "18.1"
+      - run: cargo build -p sdsearch-php --release
+      - name: Smoke test (extension loads + sdsearch_version)
+        run: php -d extension="${{ github.workspace }}\target\release\sdsearch.dll" tests\smoke.php
+      - name: Stage artifact
+        shell: pwsh
+        run: |
+          New-Item -ItemType Directory -Force -Path dist | Out-Null
+          Copy-Item target\release\sdsearch.dll dist\sdsearch-windows-x64-php84nts.dll
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdsearch-windows-x64-php84nts
+          path: dist/*
+          if-no-files-found: error
+
+  build-ubuntu:
+    name: Build + smoke (Ubuntu, PHP 8.4 NTS)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.4'
+          tools: none
+          extensions: none
+      - name: Assert PHP build is NTS
+        run: php -v | grep -q 'NTS' || { echo "::error::expected an NTS PHP build, got:"; php -v; exit 1; }
+      - name: Install clang (ext-php-rs bindgen)
+        run: sudo apt-get update && sudo apt-get install -y clang libclang-dev
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo build -p sdsearch-php --release
+      - name: Smoke test (extension loads + sdsearch_version)
+        run: php -d extension="$(pwd)/target/release/libsdsearch.so" tests/smoke.php
+      - name: Stage artifact
+        run: |
+          mkdir -p dist
+          cp target/release/libsdsearch.so dist/sdsearch-ubuntu-x64-php84nts.so
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdsearch-ubuntu-x64-php84nts
+          path: dist/*
+          if-no-files-found: error
+
+  build-el8:
+    name: Build + smoke (EL8, glibc 2.28, PHP 8.4 NTS)
+    runs-on: ubuntu-latest
+    # Build inside a Rocky 8 container (glibc 2.28) so the .so loads on RHEL/CentOS/Rocky/Alma
+    # 8 and 9. PHP 8.4 NTS dev headers come from the Remi repo. Rocky 8's glibc 2.28 is new
+    # enough for GitHub's bundled node20 actions (which is why CentOS 7 / glibc 2.17 is not an
+    # option here).
+    container: rockylinux:8
+    # ext-php-rs's build script does not auto-detect PHP inside this container even though
+    # /usr/bin/php is on PATH; point it explicitly at the Remi PHP 8.4 binary + php-config.
+    env:
+      PHP: /usr/bin/php
+      PHP_CONFIG: /usr/bin/php-config
+    steps:
+      - name: Install PHP 8.4 NTS + build toolchain
+        run: |
+          set -euxo pipefail
+          dnf install -y epel-release
+          dnf install -y https://rpms.remirepo.net/enterprise/remi-release-8.rpm
+          dnf module reset -y php
+          dnf module enable -y php:remi-8.4
+          dnf install -y php-cli php-devel clang llvm-devel gcc make git tar
+      - uses: actions/checkout@v4
+      - name: Install Rust (stable)
+        run: |
+          set -euxo pipefail
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+      - name: Assert PHP is 8.4 NTS
+        run: |
+          php -v
+          php -v | grep -q 'NTS' || { echo "::error::expected an NTS PHP build"; exit 1; }
+          php -v | grep -q '8\.4\.' || { echo "::error::expected PHP 8.4"; exit 1; }
+      - run: cargo build -p sdsearch-php --release
+      - name: Smoke test (extension loads + sdsearch_version)
+        run: php -d extension="$(pwd)/target/release/libsdsearch.so" tests/smoke.php
+      - name: Stage artifact
+        run: |
+          mkdir -p dist
+          cp target/release/libsdsearch.so dist/sdsearch-el8-x64-php84nts.so
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdsearch-el8-x64-php84nts
+          path: dist/*
+          if-no-files-found: error
+
+  publish:
+    name: Publish GitHub Release
+    runs-on: ubuntu-latest
+    needs: [build-windows, build-ubuntu, build-el8]
+    # Only publish on a tag push; workflow_dispatch stops after the builds (dry run).
+    if: startsWith(github.ref, 'refs/tags/v')
+    permissions:
+      contents: write   # create the release
+    env:
+      TAG: ${{ github.ref_name }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Assert tag matches crate version
+        run: |
+          set -euo pipefail
+          VERSION=$(grep -m1 '^version' sdsearch-core/Cargo.toml | sed -E 's/.*"(.*)".*/\1/')
+          echo "crate version: $VERSION, tag: $TAG"
+          if [ "v$VERSION" != "$TAG" ]; then
+            echo "::error::tag $TAG does not match crate version v$VERSION (bump sdsearch-core/Cargo.toml or fix the tag)"
+            exit 1
+          fi
+      - name: Download all build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          merge-multiple: true
+      - name: List artifacts
+        run: ls -la artifacts
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: artifacts/*
+          fail_on_unmatched_files: true
+          generate_release_notes: true
+          body: |
+            PHP 8.4 NTS native extension binaries.
+
+            | File | Platform | glibc | Loads on |
+            |---|---|---|---|
+            | `sdsearch-windows-x64-php84nts.dll` | Windows x64 | — | Windows, PHP 8.4 NTS |
+            | `sdsearch-ubuntu-x64-php84nts.so` | Linux x64 | ~2.39 | Ubuntu 22.04+/Debian recent, PHP 8.4 NTS |
+            | `sdsearch-el8-x64-php84nts.so` | Linux x64 | 2.28 | RHEL/CentOS/Rocky/Alma 8 & 9 (+ recent Ubuntu), PHP 8.4 NTS |

ARCHITECTURE.md

@@ -22,6 +22,10 @@ The crate is split in two:
 
 ## Format modules (`sdsearch-core/src/zsl/`)
 
+> For the byte-level layout of each file type (field offsets, encodings, worked hex
+> examples), see [`docs/FORMAT.md`](docs/FORMAT.md). This section maps those file types to
+> the modules that parse and serialize them.
+
 The reader is organized as one module per piece of the ZSL on-disk format, each a
 self-contained parser for that file type:
 

Cargo.toml

@@ -1,3 +1,18 @@
 [workspace]
 resolver = "2"
 members = ["sdsearch-core", "sdsearch-php"]
+
+# Release profile for the shipped binaries (also used by CI's --release builds, so CI
+# smoke-tests exactly what we ship).
+#
+# INVARIANT: do NOT set `panic = "abort"`. The PHP FFI boundary in sdsearch-php wraps every
+# call in catch_unwind to turn a Rust panic into a catchable PhpException and keep the PHP
+# worker alive. `panic = "abort"` would make catch_unwind catch nothing and abort the whole
+# process on any panic. It must stay the default (`unwind`).
+#
+# Also do NOT build with `-C target-cpu=native` for distributed binaries: it bakes the build
+# runner's CPU features in and can SIGILL on older deployment CPUs. Keep the x86-64 baseline.
+[profile.release]
+strip = "symbols"   # drop the symbol table: smaller binary, no internal-name leakage
+lto = "fat"         # whole-program LTO: maximum optimization (build time is not a concern)
+codegen-units = 1   # let the optimizer work across the whole crate

README.md

@@ -39,7 +39,10 @@ feature set.
   crashed worker).
 
 See [`ARCHITECTURE.md`](ARCHITECTURE.md) for the module layout, the reader/writer design,
-and the byte-fidelity details that make the format compatibility work.
+and the byte-fidelity details that make the format compatibility work, and
+[`docs/FORMAT.md`](docs/FORMAT.md) for the byte-level layout of every ZSL index file. The
+PHP API is documented in [`docs/API.md`](docs/API.md) (usage examples) and
+[`sdsearch.stub.php`](sdsearch.stub.php) (signatures + PHPDoc for IDEs / PHPStan).
 
 ## Building
 

docs/API.md

@@ -0,0 +1,162 @@
+# PHP API
+
+The `sdsearch` extension exposes three symbols to PHP: the function `sdsearch_version()`
+and the classes `SdSearch\Engine` (search) and `SdSearch\Writer` (indexing). All data
+crosses the boundary as JSON strings.
+
+The full signatures with PHPDoc live in [`sdsearch.stub.php`](../sdsearch.stub.php) at the
+repo root — point your IDE / PHPStan at it for autocompletion and type-checking (it is a
+stub, never loaded at runtime). This page is the narrative guide with runnable examples.
+
+> **Error handling.** Every method throws a catchable `\Exception` on any failure
+> (malformed JSON, missing index, lock contention, internal error). The FFI boundary is
+> panic-safe: an internal Rust panic becomes an `\Exception`, never a crashed PHP worker.
+> Wrap calls in `try/catch`.
+
+## Loading the extension
+
+```ini
+; php.ini
+extension=sdsearch.so     ; Linux
+; extension=sdsearch.dll  ; Windows
+```
+
+```php
+echo sdsearch_version(); // "0.1.0" — also a smoke test that the extension loaded
+```
+
+## Method reference
+
+### `SdSearch\Engine`
+
+| Method | Purpose | Throws |
+|---|---|---|
+| `__construct()` | Create an engine. | — |
+| `search(string $indexDir, string $paramsJson): string` | Run a query, return hits as JSON. | bad params JSON, missing index, engine error |
+
+### `SdSearch\Writer`
+
+| Method | Purpose | Throws |
+|---|---|---|
+| `__construct()` | Create a writer (not yet open). | — |
+| `open(string $indexDir): void` | Take the write-lock + open. | index locked, open error |
+| `try_open(string $indexDir): bool` | Like `open` but returns `false` if busy. | any error except "locked" |
+| `find_doc_id(string $idField, string $value): int` | `<idField>_key:value` → global doc id, or `-1`. | writer not open |
+| `find_doc_ids(string $field, string $value): int[]` | Literal `<field>:value` → all matching doc ids. | writer not open |
+| `delete_document(int $docId): void` | Mark a doc deleted (neg/out-of-range = no-op). | writer not open |
+| `add_document(string $docJson): void` | Buffer a doc for the next commit. | bad JSON, unknown kind, writer closed |
+| `commit(): int` | Flush adds+deletes, **consume** the writer. | writer not open |
+| `optimize(): int` | Commit then merge into one segment, **consume**. | writer not open |
+| `document_count(): int` | Live base + buffered − deletes. | writer not open |
+
+`commit()` and `optimize()` consume the writer: after either, the object is closed and any
+further method throws "writer not open". Create a fresh `Writer` for the next batch.
+
+## Indexing (write path)
+
+```php
+use SdSearch\Writer;
+
+$indexDir = '/var/lib/app/search-index';
+
+$w = new Writer();
+$w->open($indexDir);              // throws if another writer holds the lock
+
+// Add a document. Field kinds: "text" (tokenized+stored), "keyword" (exact+stored),
+// "unindexed" (stored only).
+$doc = [
+    'fields' => [
+        ['name' => 'id_key', 'value' => '42',            'kind' => 'keyword'],
+        ['name' => 'title',  'value' => 'How to reset a password', 'kind' => 'text'],
+        ['name' => 'body',   'value' => 'Open settings, ...',      'kind' => 'text'],
+        ['name' => 'status', 'value' => 'published',      'kind' => 'keyword'],
+    ],
+];
+$w->add_document(json_encode($doc));
+
+$w->commit();                     // flush; writer is now closed
+```
+
+### Updating an existing document
+
+Deletes are by internal doc id, so resolve first, then delete, then add the new version in
+the same batch:
+
+```php
+$w = new Writer();
+$w->open($indexDir);
+
+$docId = $w->find_doc_id('id', '42');   // resolves id_key:42 → global id, or -1
+if ($docId !== -1) {
+    $w->delete_document($docId);
+}
+$w->add_document(json_encode($updatedDoc));
+
+$w->optimize();                          // commit + compact into a single segment
+```
+
+### Non-blocking open for a background feed
+
+```php
+$w = new Writer();
+if (!$w->try_open($indexDir)) {
+    // another worker is writing — skip this cycle instead of throwing/blocking
+    return;
+}
+// ... add/delete ...
+$w->commit();
+```
+
+## Searching (read path)
+
+```php
+use SdSearch\Engine;
+
+$engine = new Engine();
+
+$params = [
+    'text'      => 'reset password',
+    'where'     => [
+        ['field' => 'status', 'values' => ['published'], 'occur' => 'must'],
+    ],
+    'in'        => [
+        ['field' => 'category_key', 'values' => ['10', '11']],
+    ],
+    'min_score' => 0.0,
+    'limit'     => 20,
+];
+
+$json = $engine->search($indexDir, json_encode($params));
+$hits = json_decode($json, true);
+
+foreach ($hits as $hit) {
+    // $hit = ['id' => int, 'score' => float, 'fields' => ['name' => 'value', ...]]
+    printf("#%d  score=%.3f  %s\n", $hit['id'], $hit['score'], $hit['fields']['title'] ?? '');
+}
+```
+
+### Query parameters
+
+| Key | Type | Meaning |
+|---|---|---|
+| `text` | string | Free-text query over tokenized fields. |
+| `where` | array | Each `{field, values[], occur}`; `occur` ∈ `must` \| `mustnot` \| `should` (default `should`). |
+| `in` | array | Each `{field, values[]}`; matches the (literal, key-suffixed) field against any value. |
+| `min_score` | float | Drop hits below this score. |
+| `limit` | int | Maximum hits to return. |
+
+Each hit is `{ "id": int, "score": float, "fields": { name: value, ... } }`, where `id` is
+the global internal document id and `fields` are the document's stored fields.
+
+## Wrapping it safely
+
+```php
+try {
+    $json = (new Engine())->search($indexDir, json_encode($params));
+    $hits = json_decode($json, true, flags: JSON_THROW_ON_ERROR);
+} catch (\Exception $e) {
+    // missing index, malformed params, or internal engine error — never a crashed worker
+    error_log('sdsearch: ' . $e->getMessage());
+    $hits = [];
+}
+```