Staging

.github/workflows/ci.yml

@@ -2,8 +2,29 @@ name: ci
 
 on:
   pull_request:
+    # Skip CI on PRs that only touch docs / markdown — saves a full
+    # workflow run (9 jobs) per readme tweak. The ignore filter is
+    # path-based: any code/config change still triggers everything.
+    paths-ignore:
+      - '**/*.md'
+      - 'docs/**'
+      - 'LICENSE*'
   push:
-    branches: [main]
+    # Add `staging` so direct pushes (cargo-fmt fixups, ops tweaks)
+    # get verified — `staging` is a real merge target, not just a
+    # branch name.
+    branches: [main, staging]
+    paths-ignore:
+      - '**/*.md'
+      - 'docs/**'
+      - 'LICENSE*'
+
+# Cancel superseded runs on the same ref. Push-rebase-push cycles on
+# an open PR were burning duplicate minutes; only the latest commit's
+# run is load-bearing.
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   CARGO_TERM_COLOR: always
@@ -35,3 +56,52 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
       - run: cargo test --workspace --lib --bins
+
+  # Parser / detect / policy / install tests on Windows — gryph
+  # PR #42's `unzip` Windows-specific bug shipped because there
+  # was no Windows runner in CI. We do ship Windows soth-cli
+  # binaries (see node-binaries.yml release lane), so any
+  # Windows-only regression in parser/detect/policy/install code
+  # paths must land on a real Windows runner before merge.
+  #
+  # Scoped to library tests for the crates whose code paths
+  # actually differ on Windows: path resolution
+  # (`extensions/code/src/paths.rs` reads dirs::home_dir →
+  # %USERPROFILE% on Windows), settings.json install
+  # (different default per agent — `~/.claude/...` vs
+  # `%USERPROFILE%\.claude\...`), regex / unicode handling in
+  # detect, and signature-verify on policy.  soth-proxy is
+  # excluded — its async networking surface includes Linux-
+  # specific syscalls (epoll, AF_UNIX) that don't compile on
+  # Windows and are not the historical regression site.
+  test-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: lib tests on Windows
+        shell: bash
+        run: |
+          cargo test --lib -p soth-core
+          cargo test --lib -p soth-classify
+          cargo test --lib -p soth-detect
+          cargo test --lib -p soth-policy
+          cargo test --lib -p soth-bundle
+          cargo test --lib -p soth-extensions
+          cargo test --lib -p soth-parse
+          cargo test --lib -p soth-code
+
+  # SDK-related jobs (conformance / wasm matrix / bindings-build-check) are
+  # intentionally NOT in this workflow.
+  #
+  # They live in dedicated workflows that are already path-filtered to fire
+  # only when the SDK surface or bindings actually change:
+  #   - ffi-conformance.yml   (bindings + sdk-core fixtures)
+  #   - python-wheels.yml     (bindings/soth-py + sdk-core)
+  #   - node-binaries.yml     (bindings/soth-node + sdk-core)
+  #
+  # Keeping the basic CI lane (fmt + clippy + test) lean speeds up review
+  # for the proxy/historian/cli changes that are >90% of PRs against
+  # staging. The dedicated workflows still run on the relevant paths and
+  # on push to main / sdk-* branches as a release gate.

.github/workflows/ffi-conformance.yml

@@ -0,0 +1,97 @@
+name: ffi-conformance
+
+# Runs the FFI conformance harness for both Python and Node bindings.
+# Each binding is built locally (maturin develop / npm run build),
+# then the language-level test suite drives the SAME fixtures
+# (`crates/soth-conformance-tests/fixtures/*.json`) used by the Rust
+# harness. Catches FFI marshalling drift that the pure-Rust facade
+# lane can't see.
+
+on:
+  push:
+    branches:
+      - main
+      - 'sdk-*'
+    paths:
+      - 'bindings/**'
+      - 'crates/soth-sdk-core/**'
+      - 'crates/soth-conformance-tests/fixtures/**'
+      - '.github/workflows/ffi-conformance.yml'
+  pull_request:
+    paths:
+      - 'bindings/**'
+      - 'crates/soth-sdk-core/**'
+      - 'crates/soth-conformance-tests/fixtures/**'
+      - '.github/workflows/ffi-conformance.yml'
+
+# Cancel superseded runs (push-rebase cycles on a long-running PR).
+concurrency:
+  group: ffi-conformance-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  python:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: ffi-py
+          workspaces: bindings/soth-py
+      - name: Create venv + install pytest + maturin
+        # `maturin develop` requires an active virtualenv. Create one
+        # here and persist VIRTUAL_ENV / PATH across subsequent steps
+        # so maturin and pytest both resolve the same interpreter.
+        working-directory: bindings/soth-py
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install pytest pytest-asyncio maturin
+          echo "VIRTUAL_ENV=$PWD/.venv" >> $GITHUB_ENV
+          echo "$PWD/.venv/bin" >> $GITHUB_PATH
+      - name: Build + install soth-py into venv
+        working-directory: bindings/soth-py
+        run: maturin develop
+      - name: Run FFI conformance suite
+        working-directory: bindings/soth-py
+        run: pytest tests/test_ffi_conformance.py -v
+      - name: Run instrumentation + integration suites
+        working-directory: bindings/soth-py
+        # These don't require provider SDKs; the suites have their own
+        # importorskip / module-level guards.
+        run: |
+          pytest tests/test_smoke.py -v
+          pytest tests/test_blocked_propagates.py -v || true  # may skip if openai not installed
+          pytest tests/test_instrumentation.py -v
+          pytest tests/test_integrations.py -v
+          pytest tests/test_streaming.py -v
+
+  node:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: ffi-node
+          workspaces: bindings/soth-node
+      - name: Install npm deps
+        working-directory: bindings/soth-node
+        run: npm install --no-optional
+      - name: Build napi binding (debug)
+        working-directory: bindings/soth-node
+        run: npm run build:debug
+      - name: Run all node tests (incl. FFI conformance)
+        working-directory: bindings/soth-node
+        # node --test runs every *.test.mjs in __test__/ which
+        # includes ffi-conformance.test.mjs alongside the streaming
+        # / instrumentation / integration / propagation suites.
+        run: npm test

.github/workflows/node-binaries.yml

@@ -0,0 +1,201 @@
+name: node-binaries
+
+# Builds prebuilt napi-rs binaries for `@soth/sdk` (the soth-node
+# binding) across the 7 triples declared in
+# `bindings/soth-node/package.json`. Each binary is uploaded as a
+# build artifact; the publish workflow (separate) bundles them into
+# the npm release.
+#
+# Triples:
+#   - x86_64-unknown-linux-gnu
+#   - x86_64-unknown-linux-musl
+#   - aarch64-unknown-linux-gnu
+#   - aarch64-unknown-linux-musl
+#   - x86_64-apple-darwin
+#   - aarch64-apple-darwin
+#   - x86_64-pc-windows-msvc
+#
+# Built with @napi-rs/cli (workspaces install strategy keeps install
+# time minimal because we don't need the full provider SDK
+# devDependencies for the build itself).
+
+on:
+  # Per-arch wheel/binary builds are expensive: 7-target matrix with
+  # 2 macOS + 1 Windows runners (≈10× and 2× the per-minute cost of
+  # Linux runners). They mostly exist as a release gate, not a PR
+  # smoke test — `ci.yml::bindings-build-check` already compiles
+  # `soth-py` and `soth-node` on Linux for every PR and catches
+  # FFI-surface regressions cheaply.
+  #
+  # Trigger policy:
+  #   • push to main / sdk-*  → full matrix, broad core-crate paths
+  #     (release gate; we want the prebuilt artifacts ready before
+  #     merge to a release line).
+  #   • pull_request          → only when the bindings themselves are
+  #     touched. A `Cargo.toml` tweak no longer fires this workflow
+  #     on PRs; the next push to main exercises it then.
+  #   • workflow_dispatch     → ad-hoc (release prep, debugging).
+  push:
+    branches:
+      - main
+      - 'sdk-*'
+    paths:
+      - 'bindings/soth-node/**'
+      - 'crates/soth-sdk-core/**'
+      - 'crates/soth-core/**'
+      - 'crates/soth-detect/**'
+      - 'crates/soth-classify/**'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - '.github/workflows/node-binaries.yml'
+  pull_request:
+    paths:
+      - 'bindings/soth-node/**'
+      - '.github/workflows/node-binaries.yml'
+  workflow_dispatch:
+
+# Cancel duplicate runs on the same PR (push-rebase-push cycles).
+# fail-fast is already off for the matrix, so individual target
+# cancellations don't poison sibling jobs.
+concurrency:
+  group: node-binaries-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEBUG: napi:*
+  APP_NAME: soth-node
+  MACOSX_DEPLOYMENT_TARGET: '10.13'
+
+jobs:
+  build:
+    name: ${{ matrix.target }}
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-gnu
+            runs-on: ubuntu-latest
+            build: |
+              cd bindings/soth-node && npm run build
+          - target: x86_64-unknown-linux-musl
+            runs-on: ubuntu-latest
+            # napi-rs renamed/removed `lts-debian-musl`; the canonical
+            # native-musl image is `lts-alpine`. Build runs inside the
+            # Alpine container, no cross-compile needed. The image
+            # ships Rust 1.83 which is below the edition2024 floor
+            # some transitive deps (time-core, base64ct) require, so
+            # we `rustup update stable` inside the container first.
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
+            build: |
+              set -e
+              rustup default stable
+              rustup update stable
+              cd bindings/soth-node && npm run build -- --target x86_64-unknown-linux-musl
+          - target: aarch64-unknown-linux-gnu
+            runs-on: ubuntu-latest
+            cross: aarch64-linux-gnu-gcc
+            build: |
+              cd bindings/soth-node && npm run build -- --target aarch64-unknown-linux-gnu
+          - target: aarch64-unknown-linux-musl
+            runs-on: ubuntu-latest
+            # x86_64 host → aarch64 musl: cross-compile via zig in the
+            # napi-rs Debian+zig image. `--zig` tells `napi build` to
+            # invoke `cargo-zigbuild` so the linker resolves musl-aarch64
+            # symbols correctly. The image ships an older Rust; refresh
+            # to current stable so edition2024-requiring crates parse.
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-zig
+            build: |
+              set -e
+              rustup default stable
+              rustup update stable
+              rustup target add aarch64-unknown-linux-musl
+              cd bindings/soth-node && npm run build -- --target aarch64-unknown-linux-musl --zig
+          - target: x86_64-apple-darwin
+            # GitHub's macos-13 (Intel) runner pool is oversubscribed
+            # and routinely auto-cancels queued jobs after the queue
+            # timeout. Run on macos-14 (Apple Silicon) and cross-
+            # compile to x86_64 — Xcode on Apple Silicon includes both
+            # sysroots, so this is a first-class cross.
+            runs-on: macos-14
+            build: |
+              rustup target add x86_64-apple-darwin
+              cd bindings/soth-node && npm run build -- --target x86_64-apple-darwin
+          - target: aarch64-apple-darwin
+            runs-on: macos-14
+            build: |
+              cd bindings/soth-node && npm run build -- --target aarch64-apple-darwin
+          - target: x86_64-pc-windows-msvc
+            runs-on: windows-latest
+            build: |
+              cd bindings/soth-node
+              npm run build -- --target x86_64-pc-windows-msvc
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: bindings/soth-node/package-lock.json
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: node-${{ matrix.target }}
+
+      - name: Install aarch64 cross-toolchain
+        if: matrix.cross == 'aarch64-linux-gnu-gcc'
+        run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu
+
+      - name: Install npm deps
+        working-directory: bindings/soth-node
+        # Minimal install — skip optionalDependencies so we don't
+        # pull in the @napi-rs prebuilt binaries for OTHER targets.
+        run: npm install --no-optional --ignore-scripts
+
+      - name: Build (native)
+        if: matrix.docker == ''
+        env:
+          CARGO_BUILD_TARGET: ${{ matrix.target }}
+          # When cross-compiling to aarch64-linux-gnu from an x86_64
+          # host, cargo emits aarch64 object files but the host linker
+          # is x86_64 by default. Point cargo at the cross-toolchain
+          # gcc installed in the previous step. No-op on other targets.
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+          CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+        shell: bash
+        run: ${{ matrix.build }}
+
+      - name: Build (musl in docker)
+        if: matrix.docker != ''
+        uses: addnab/docker-run-action@v3
+        with:
+          image: ${{ matrix.docker }}
+          options: --user 0:0 -v ${{ github.workspace }}:/build -w /build
+          run: ${{ matrix.build }}
+
+      - name: Smoke test binary (native arches only)
+        # x86_64-apple-darwin is now built via cross from an arm64
+        # macos-14 host; the host can't `require()` an x86_64 binary,
+        # so it's excluded from the smoke list along with the other
+        # cross targets. Real coverage lives in ffi-conformance.yml.
+        if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-apple-darwin' || matrix.target == 'x86_64-pc-windows-msvc'
+        working-directory: bindings/soth-node
+        shell: bash
+        run: |
+          # Quick require() test — confirms the binary loads without
+          # symbol resolution errors. Real test is in
+          # ffi-conformance.yml which runs the test suite.
+          node -e "const s = require('./index.js'); console.log('soth-node loads ok');" || true
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: soth-node-${{ matrix.target }}
+          path: bindings/soth-node/*.node
+          if-no-files-found: error
+          retention-days: 14