diff --git a/.github/scripts/install-zig.sh b/.github/scripts/install-zig.sh deleted file mode 100644 index 8410fee..0000000 --- a/.github/scripts/install-zig.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ "$#" -ne 1 ]; then - echo "usage: $0 " >&2 - exit 1 -fi - -version="$1" - -python_bin="${PYTHON:-python3}" -if ! command -v "$python_bin" >/dev/null 2>&1; then - python_bin="python" -fi -if ! command -v "$python_bin" >/dev/null 2>&1; then - echo "python is required to install Zig" >&2 - exit 1 -fi - -runner_os="${RUNNER_OS:-$(uname -s)}" -runner_arch="${RUNNER_ARCH:-$(uname -m)}" - -case "$runner_os" in - Linux | linux) - zig_os="linux" - ;; - Darwin | macOS) - zig_os="macos" - ;; - Windows | MINGW* | MSYS* | CYGWIN*) - zig_os="windows" - ;; - *) - echo "unsupported runner OS: $runner_os" >&2 - exit 1 - ;; -esac - -case "$runner_arch" in - X64 | x86_64 | amd64) - zig_arch="x86_64" - ;; - ARM64 | arm64 | aarch64) - zig_arch="aarch64" - ;; - *) - echo "unsupported runner architecture: $runner_arch" >&2 - exit 1 - ;; -esac - -host_key="${zig_arch}-${zig_os}" -tool_root="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/nullwatch-zig" -install_dir="${tool_root}/${version}/${host_key}" -zig_bin="zig" -if [ "$zig_os" = "windows" ]; then - zig_bin="zig.exe" -fi - -if [ ! -x "${install_dir}/${zig_bin}" ]; then - mkdir -p "$(dirname "$install_dir")" - - zig_metadata="$( - "$python_bin" - "$version" "$host_key" <<'PY' -import json -import sys -import urllib.request - -version = sys.argv[1] -host_key = sys.argv[2] - -with urllib.request.urlopen("https://ziglang.org/download/index.json") as response: - data = json.load(response) - -host = data.get(version, {}).get(host_key) -if not host: - raise SystemExit(f"missing Zig download metadata for version={version!r} host={host_key!r}") - -archive_url = host.get("tarball") or host.get("zip") -checksum = host.get("shasum") or "" -if not archive_url: - raise SystemExit(f"missing archive URL for version={version!r} host={host_key!r}") - -print(archive_url) -print(checksum) -PY - )" - - archive_url="$(printf '%s\n' "$zig_metadata" | sed -n '1p')" - expected_sha="$(printf '%s\n' "$zig_metadata" | sed -n '2p')" - if [ -z "$archive_url" ]; then - echo "failed to resolve Zig download URL" >&2 - exit 1 - fi - - archive_name="${archive_url##*/}" - archive_dir="$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/zig-archive.XXXXXX")" - archive_path="${archive_dir}/${archive_name}" - extract_dir="$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/zig-extract.XXXXXX")" - trap 'rm -rf "$archive_dir"; rm -rf "$extract_dir"' EXIT - - curl -fsSL --retry 3 --retry-all-errors "$archive_url" -o "$archive_path" - - "$python_bin" - "$archive_path" "$expected_sha" <<'PY' -import hashlib -import sys - -path = sys.argv[1] -expected = sys.argv[2].strip().lower() -if not expected: - raise SystemExit(0) - -digest = hashlib.sha256() -with open(path, "rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - -actual = digest.hexdigest().lower() -if actual != expected: - raise SystemExit(f"checksum mismatch for {path}: expected {expected}, got {actual}") -PY - - "$python_bin" - "$archive_path" "$extract_dir" <<'PY' -import pathlib -import sys -import tarfile -import zipfile - -archive = pathlib.Path(sys.argv[1]) -destination = pathlib.Path(sys.argv[2]) -destination.mkdir(parents=True, exist_ok=True) - -def ensure_within_destination(relative_name: str) -> None: - target = (destination / relative_name).resolve() - if destination.resolve() not in target.parents and target != destination.resolve(): - raise SystemExit(f"archive entry escapes destination: {relative_name}") - -if archive.suffix == ".zip": - with zipfile.ZipFile(archive) as handle: - for member in handle.namelist(): - ensure_within_destination(member) - handle.extractall(destination) -else: - with tarfile.open(archive, "r:*") as handle: - for member in handle.getnames(): - ensure_within_destination(member) - handle.extractall(destination) -PY - - extracted_dir="$(find "$extract_dir" -mindepth 1 -maxdepth 1 -type d | head -n 1)" - if [ -z "$extracted_dir" ]; then - echo "failed to extract Zig archive: $archive_url" >&2 - exit 1 - fi - - rm -rf "$install_dir" - mv "$extracted_dir" "$install_dir" -fi - -if [ -n "${GITHUB_PATH:-}" ]; then - printf '%s\n' "$install_dir" >> "$GITHUB_PATH" -else - echo "GITHUB_PATH is not set; add this directory to PATH manually: $install_dir" >&2 -fi - -"${install_dir}/${zig_bin}" version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2df5e36..e907c9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,92 +1,22 @@ name: CI -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - ZIG_VERSION: "0.16.0" - on: push: branches: [main] pull_request: branches: [main] -jobs: - test: - name: Test (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - target: linux-x86_64 - zig_target: x86_64-linux-musl - - os: macos-latest - target: macos-aarch64 - zig_target: aarch64-macos - - os: windows-latest - target: windows-x86_64 - zig_target: x86_64-windows - - steps: - - uses: actions/checkout@v6 - - - name: Install Zig 0.16.0 - run: bash .github/scripts/install-zig.sh "${ZIG_VERSION}" - - - name: Cache Zig build outputs - uses: actions/cache@v5 - with: - path: | - .zig-cache - ~/.cache/zig - key: zig-${{ matrix.target }}-${{ hashFiles('src/**/*.zig', 'build.zig', 'build.zig.zon', 'tests/test_e2e.sh') }} - restore-keys: zig-${{ matrix.target }}- - - - name: Run unit tests - run: zig build test --summary all 2>&1 | tee test-output.txt - - - name: Run E2E tests - if: runner.os == 'Linux' - run: bash tests/test_e2e.sh +permissions: + contents: read - - name: Build ReleaseSmall - run: zig build -Dtarget=${{ matrix.zig_target }} -Doptimize=ReleaseSmall -Dversion=ci - - - name: Preserve host binary - run: | - mkdir -p ci-artifacts - cp "zig-out/bin/nullwatch${{ runner.os == 'Windows' && '.exe' || '' }}" "ci-artifacts/nullwatch${{ runner.os == 'Windows' && '.exe' || '' }}" - - - name: Job summary - if: always() - run: | - echo "## nullwatch CI - ${{ matrix.target }}" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "| Metric | Value |" >> "$GITHUB_STEP_SUMMARY" - echo "|--------|-------|" >> "$GITHUB_STEP_SUMMARY" - - if [ -f test-output.txt ]; then - TESTS=$(grep -o '[0-9]*/[0-9]* tests passed' test-output.txt | head -1 || true) - echo "| Tests | ${TESTS:-n/a} |" >> "$GITHUB_STEP_SUMMARY" - fi - - BIN="ci-artifacts/nullwatch" - if [ "${{ runner.os }}" = "Windows" ]; then - BIN="ci-artifacts/nullwatch.exe" - fi - if [ -f "$BIN" ]; then - SIZE=$(wc -c < "$BIN" | tr -d ' ') - SIZE_HR=$(awk "BEGIN {printf \"%.1f MB\", $SIZE / 1048576}") - echo "| Binary (ReleaseSmall) | ${SIZE_HR} (${SIZE} bytes) |" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload binary - if: success() - uses: actions/upload-artifact@v7 - with: - name: nullwatch-${{ matrix.target }} - path: ci-artifacts/nullwatch${{ runner.os == 'Windows' && '.exe' || '' }} +jobs: + zig: + uses: nullclaw/nullbuilder/.github/workflows/zig-ci.yml@v1 + permissions: + contents: read + with: + binary_name: nullwatch + artifact_prefix: nullwatch + build_args: -Dversion=ci + e2e_target: linux-x86_64 + e2e_command: bash tests/test_e2e.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f9a20d0..a5fd1ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,134 +1,35 @@ name: Release -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - ZIG_VERSION: "0.16.0" - on: push: tags: ['v*'] workflow_dispatch: -jobs: - build: - name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - strategy: - matrix: - include: - - os: ubuntu-latest - target: linux-x86_64 - zig_target: x86_64-linux-musl - ext: "" - - os: ubuntu-latest - target: linux-aarch64 - zig_target: aarch64-linux-musl - ext: "" - - os: ubuntu-latest - target: linux-riscv64 - zig_target: riscv64-linux-musl - ext: "" - - os: macos-latest - target: macos-aarch64 - zig_target: aarch64-macos - ext: "" - - os: macos-latest - target: macos-x86_64 - zig_target: x86_64-macos - ext: "" - - os: windows-latest - target: windows-x86_64 - zig_target: x86_64-windows - ext: ".exe" - - os: windows-latest - target: windows-aarch64 - zig_target: aarch64-windows - ext: ".exe" - - steps: - - uses: actions/checkout@v6 - - - name: Install Zig 0.16.0 - run: bash .github/scripts/install-zig.sh "${ZIG_VERSION}" - - - name: Resolve build version - id: version - run: | - if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then - echo "value=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - else - echo "value=dev" >> "$GITHUB_OUTPUT" - fi - - - name: Build ReleaseSmall - run: zig build -Doptimize=ReleaseSmall -Dversion=${{ steps.version.outputs.value }} -Dtarget=${{ matrix.zig_target }} - - - name: Upload artifact - uses: actions/upload-artifact@v7 - with: - name: nullwatch-${{ matrix.target }} - path: zig-out/bin/nullwatch${{ matrix.ext }} - - source: - name: Prepare source archive - runs-on: ubuntu-latest - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v6 - - - name: Create source archive - run: | - archive_name="nullwatch-source-${GITHUB_REF_NAME}.tar.gz" - tar \ - --exclude='.git' \ - --exclude='.zig-cache' \ - --exclude='zig-out' \ - -czf "/tmp/${archive_name}" . - mv "/tmp/${archive_name}" . - echo "ARCHIVE_NAME=${archive_name}" >> "$GITHUB_ENV" - - - name: Upload source archive - uses: actions/upload-artifact@v7 - with: - name: nullwatch-source - path: ${{ env.ARCHIVE_NAME }} +permissions: + contents: read +jobs: release: - needs: [build, source] - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest + uses: nullclaw/nullbuilder/.github/workflows/zig-release.yml@v1 permissions: contents: write - - steps: - - uses: actions/download-artifact@v8 - - - name: Rename binaries - run: | - mv nullwatch-linux-x86_64/nullwatch nullwatch-linux-x86_64.bin - mv nullwatch-linux-aarch64/nullwatch nullwatch-linux-aarch64.bin - mv nullwatch-linux-riscv64/nullwatch nullwatch-linux-riscv64.bin - mv nullwatch-macos-aarch64/nullwatch nullwatch-macos-aarch64.bin - mv nullwatch-macos-x86_64/nullwatch nullwatch-macos-x86_64.bin - mv nullwatch-windows-x86_64/nullwatch.exe nullwatch-windows-x86_64.exe - mv nullwatch-windows-aarch64/nullwatch.exe nullwatch-windows-aarch64.exe - - - name: Create release - uses: softprops/action-gh-release@v2 - with: - files: | - nullwatch-linux-x86_64.bin - nullwatch-linux-aarch64.bin - nullwatch-linux-riscv64.bin - nullwatch-macos-aarch64.bin - nullwatch-macos-x86_64.bin - nullwatch-windows-x86_64.exe - nullwatch-windows-aarch64.exe - nullwatch-source/*.tar.gz - generate_release_notes: true + secrets: inherit + with: + binary_name: nullwatch + artifact_prefix: nullwatch + targets_json: >- + [ + {"os":"ubuntu-latest","target":"linux-x86_64","zig_target":"x86_64-linux-musl","zig_cpu":"","ext":""}, + {"os":"ubuntu-latest","target":"linux-aarch64","zig_target":"aarch64-linux-musl","zig_cpu":"","ext":""}, + {"os":"ubuntu-latest","target":"linux-riscv64","zig_target":"riscv64-linux-musl","zig_cpu":"","ext":""}, + {"os":"macos-latest","target":"macos-aarch64","zig_target":"aarch64-macos","zig_cpu":"","ext":""}, + {"os":"macos-latest","target":"macos-x86_64","zig_target":"x86_64-macos","zig_cpu":"","ext":""}, + {"os":"windows-latest","target":"windows-x86_64","zig_target":"x86_64-windows","zig_cpu":"","ext":".exe"}, + {"os":"windows-latest","target":"windows-aarch64","zig_target":"aarch64-windows","zig_cpu":"","ext":".exe"} + ] + source_archive: true + source_archive_name: nullwatch-source-${{ github.ref_name }}.tar.gz + source_archive_excludes: | + .git + .zig-cache + zig-out diff --git a/README.md b/README.md index bd9aaf6..4a79f52 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,19 @@ List evals: zig build run -- evals --dataset prod-shadow --verdict fail ``` +Seed local demo runs: + +```bash +zig build run -- demo-seed +zig build run -- runs --limit 20 +zig build run -- run demo-tool-failure +``` + +`demo-seed` creates a deterministic, idempotent local dataset for demos and +manual testing without API keys, hosted services, or a running agent workload. +It includes a passing code-review run, a failed tool-call run, and a +handoff/retry run with checkpoint context. + Ingest a span from the CLI: ```bash @@ -354,16 +367,28 @@ zig build run -- --from-json '{"home":"~/.nullwatch","port":7710,"data_dir":"dat This keeps the service headless while letting `nullhub` own install/setup UI. +For a local NullHub flight-recorder demo: + +```bash +zig build run -- demo-seed +zig build run -- serve --port 7710 +``` + +Start NullHub with `NULLWATCH_URL=http://127.0.0.1:7710` and open the +Observability page to inspect the seeded runs, spans, evals, token usage, cost, +and failure context. + ## CI and releases - `tests/test_e2e.sh` boots a real server and validates auth, ingest, OTLP mapping, and CLI queries. -- `.github/workflows/ci.yml` runs unit tests, Linux E2E, and host builds on Linux/macOS/Windows. -- `.github/workflows/release.yml` builds tagged release artifacts for Linux, macOS, and Windows and publishes them to GitHub Releases. +- `.github/workflows/ci.yml` delegates unit tests, Linux E2E, and host builds to `nullclaw/nullbuilder`. +- `.github/workflows/release.yml` delegates tagged release artifacts for Linux, macOS, and Windows to `nullclaw/nullbuilder`. - `scripts/build-release.sh` produces the same release artifact names locally plus `SHA256SUMS`. ## Near-term next steps - Replace JSONL storage with embedded SQLite while preserving the API contract. +- Extend demo fixtures with GenAI/OpenInference attributes and scenario selection. - Add dataset, prompt version, and experiment entities. - Add regression diff endpoints for comparing prompt/model/strategy versions. - Add alert rules and anomaly summaries that `nullhub` can render. diff --git a/src/demo_seed.zig b/src/demo_seed.zig new file mode 100644 index 0000000..dfb8e23 --- /dev/null +++ b/src/demo_seed.zig @@ -0,0 +1,283 @@ +const std = @import("std"); +const domain = @import("domain.zig"); +const Store = @import("store.zig").Store; + +pub const SeedSummary = struct { + status: []const u8 = "ok", + runs_created: usize = 0, + runs_skipped: usize = 0, + spans_created: usize = 0, + evals_created: usize = 0, +}; + +const base_ms: i64 = 1_710_000_000_000; + +pub fn seed(allocator: std.mem.Allocator, store: *Store) !SeedSummary { + var summary = SeedSummary{}; + + try seedReviewPass(allocator, store, &summary); + try seedToolFailure(allocator, store, &summary); + try seedHandoffRetry(allocator, store, &summary); + + return summary; +} + +fn runExists(allocator: std.mem.Allocator, store: *Store, run_id: []const u8) !bool { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const detail = try store.getRunDetail(arena.allocator(), run_id); + return detail != null; +} + +fn seedReviewPass(allocator: std.mem.Allocator, store: *Store, summary: *SeedSummary) !void { + const run_id = "demo-code-review-pass"; + if (try runExists(allocator, store, run_id)) { + summary.runs_skipped += 1; + return; + } + + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-review-pass", + .span_id = "review-pass-root", + .source = "nullclaw", + .operation = "agent.run", + .status = "ok", + .started_at_ms = base_ms, + .ended_at_ms = base_ms + 1420, + .agent_id = "reviewer-1", + .task_id = "ticket-demo-101", + .attributes_json = "{\"pipeline_id\":\"code-review\",\"stage\":\"review\"}", + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-review-pass", + .span_id = "review-pass-model", + .parent_span_id = "review-pass-root", + .source = "nullclaw", + .operation = "model.call", + .status = "ok", + .started_at_ms = base_ms + 80, + .ended_at_ms = base_ms + 940, + .agent_id = "reviewer-1", + .model = "gpt-5-mini", + .input_tokens = 1280, + .output_tokens = 420, + .cost_usd = 0.013, + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-review-pass", + .span_id = "review-pass-transition", + .parent_span_id = "review-pass-root", + .source = "nulltickets", + .operation = "tracker.transition", + .status = "ok", + .started_at_ms = base_ms + 1020, + .ended_at_ms = base_ms + 1180, + .task_id = "ticket-demo-101", + .attributes_json = "{\"from\":\"review\",\"to\":\"done\",\"trigger\":\"approve\"}", + }); + try addEval(store, summary, .{ + .run_id = run_id, + .eval_key = "review_quality", + .scorer = "demo-rubric", + .score = 0.94, + .verdict = "pass", + .dataset = "flight-recorder-demo", + .notes = "Review found the intended issue and approved after tests.", + .recorded_at_ms = base_ms + 1500, + }); + + summary.runs_created += 1; +} + +fn seedToolFailure(allocator: std.mem.Allocator, store: *Store, summary: *SeedSummary) !void { + const run_id = "demo-tool-failure"; + if (try runExists(allocator, store, run_id)) { + summary.runs_skipped += 1; + return; + } + + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-tool-failure", + .span_id = "tool-failure-root", + .source = "nullboiler", + .operation = "workflow.step", + .status = "error", + .started_at_ms = base_ms + 10_000, + .ended_at_ms = base_ms + 13_840, + .agent_id = "coder-1", + .task_id = "ticket-demo-202", + .error_message = "workflow failed after shell tool error", + .attributes_json = "{\"workflow_id\":\"bug-fix\",\"node\":\"run-tests\"}", + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-tool-failure", + .span_id = "tool-failure-model", + .parent_span_id = "tool-failure-root", + .source = "nullclaw", + .operation = "model.call", + .status = "ok", + .started_at_ms = base_ms + 10_120, + .ended_at_ms = base_ms + 11_050, + .agent_id = "coder-1", + .model = "gpt-5-mini", + .input_tokens = 2140, + .output_tokens = 620, + .cost_usd = 0.022, + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-tool-failure", + .span_id = "tool-failure-shell", + .parent_span_id = "tool-failure-root", + .source = "nullclaw", + .operation = "tool.call", + .status = "error", + .started_at_ms = base_ms + 11_100, + .ended_at_ms = base_ms + 13_600, + .agent_id = "coder-1", + .tool_name = "shell", + .error_message = "zig build test exited with status 1", + .attributes_json = "{\"command\":\"zig build test --summary all\",\"exit_code\":1}", + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-tool-failure", + .span_id = "tool-failure-event", + .parent_span_id = "tool-failure-root", + .source = "nulltickets", + .operation = "run.event", + .status = "ok", + .started_at_ms = base_ms + 13_660, + .ended_at_ms = base_ms + 13_760, + .task_id = "ticket-demo-202", + .attributes_json = "{\"kind\":\"test_failure\",\"artifact\":\"zig-test-output.txt\"}", + }); + try addEval(store, summary, .{ + .run_id = run_id, + .eval_key = "tool_success", + .scorer = "demo-rubric", + .score = 0.31, + .verdict = "fail", + .dataset = "flight-recorder-demo", + .notes = "The workflow surfaced a failing shell tool call with enough context to debug.", + .recorded_at_ms = base_ms + 14_000, + }); + + summary.runs_created += 1; +} + +fn seedHandoffRetry(allocator: std.mem.Allocator, store: *Store, summary: *SeedSummary) !void { + const run_id = "demo-handoff-retry"; + if (try runExists(allocator, store, run_id)) { + summary.runs_skipped += 1; + return; + } + + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-handoff-retry", + .span_id = "handoff-root", + .source = "nullboiler", + .operation = "workflow.run", + .status = "ok", + .started_at_ms = base_ms + 20_000, + .ended_at_ms = base_ms + 24_500, + .task_id = "ticket-demo-303", + .attributes_json = "{\"workflow_id\":\"feature-dev\",\"checkpoint_count\":3}", + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-handoff-retry", + .span_id = "handoff-analyst", + .parent_span_id = "handoff-root", + .source = "nullclaw", + .operation = "agent.handoff", + .status = "ok", + .started_at_ms = base_ms + 20_100, + .ended_at_ms = base_ms + 21_150, + .agent_id = "analyst-1", + .attributes_json = "{\"to_agent\":\"coder-1\",\"reason\":\"implementation required\"}", + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-handoff-retry", + .span_id = "handoff-coder", + .parent_span_id = "handoff-root", + .source = "nullclaw", + .operation = "agent.handoff", + .status = "ok", + .started_at_ms = base_ms + 21_220, + .ended_at_ms = base_ms + 23_800, + .agent_id = "coder-1", + .attributes_json = "{\"to_agent\":\"reviewer-1\",\"reason\":\"ready for review\",\"retry\":1}", + }); + try addSpan(store, summary, .{ + .run_id = run_id, + .trace_id = "trace-demo-handoff-retry", + .span_id = "handoff-checkpoint", + .parent_span_id = "handoff-root", + .source = "nullboiler", + .operation = "checkpoint.created", + .status = "ok", + .started_at_ms = base_ms + 23_920, + .ended_at_ms = base_ms + 24_020, + .attributes_json = "{\"checkpoint_id\":\"cp-demo-303-3\",\"node\":\"review\"}", + }); + try addEval(store, summary, .{ + .run_id = run_id, + .eval_key = "handoff_budget", + .scorer = "demo-rubric", + .score = 0.78, + .verdict = "pass", + .dataset = "flight-recorder-demo", + .notes = "The handoff chain stayed within the expected retry budget.", + .recorded_at_ms = base_ms + 24_800, + }); + + summary.runs_created += 1; +} + +fn addSpan(store: *Store, summary: *SeedSummary, payload: domain.SpanIngest) !void { + _ = try store.ingestSpan(payload); + summary.spans_created += 1; +} + +fn addEval(store: *Store, summary: *SeedSummary, payload: domain.EvalIngest) !void { + _ = try store.ingestEval(payload); + summary.evals_created += 1; +} + +test "demo seed creates local observability runs and is idempotent" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const tmp_dir = @import("compat.zig").fs.Dir.wrap(tmp.dir); + const data_dir = try tmp_dir.realpathAlloc(std.testing.allocator, "."); + defer std.testing.allocator.free(data_dir); + + var store = try Store.init(std.testing.allocator, data_dir); + defer store.deinit(); + + const first = try seed(std.testing.allocator, &store); + try std.testing.expectEqual(@as(usize, 3), first.runs_created); + try std.testing.expectEqual(@as(usize, 0), first.runs_skipped); + try std.testing.expectEqual(@as(usize, 11), first.spans_created); + try std.testing.expectEqual(@as(usize, 3), first.evals_created); + + const second = try seed(std.testing.allocator, &store); + try std.testing.expectEqual(@as(usize, 0), second.runs_created); + try std.testing.expectEqual(@as(usize, 3), second.runs_skipped); + + const summary = try store.getSystemSummary(std.testing.allocator); + try std.testing.expectEqual(@as(usize, 3), summary.run_count); + try std.testing.expectEqual(@as(usize, 11), summary.span_count); + try std.testing.expectEqual(@as(usize, 3), summary.eval_count); + try std.testing.expectEqual(@as(usize, 2), summary.error_count); + try std.testing.expectEqual(@as(usize, 1), summary.fail_count); +} diff --git a/src/main.zig b/src/main.zig index f82a130..701695a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2,6 +2,7 @@ const std = @import("std"); const std_compat = @import("compat.zig"); const api = @import("api.zig"); const config = @import("config.zig"); +const demo_seed = @import("demo_seed.zig"); const domain = @import("domain.zig"); const Store = @import("store.zig").Store; const version = @import("version.zig"); @@ -138,6 +139,13 @@ pub fn main(init: std.process.Init) !void { return; } + if (std.mem.eql(u8, command, "demo-seed")) { + var parsed = try parseCommonArgs(allocator, &cursor); + defer parsed.deinit(allocator); + try runDemoSeedCommand(allocator, parsed.runtime); + return; + } + std.debug.print("unknown command: {s}\n\n", .{command}); printUsage(); std.process.exit(1); @@ -328,6 +336,14 @@ fn runEvalIngestCommand(allocator: std.mem.Allocator, runtime: RuntimeConfig, js try writeJsonToStdout(allocator, record); } +fn runDemoSeedCommand(allocator: std.mem.Allocator, runtime: RuntimeConfig) !void { + var store = try Store.init(allocator, runtime.data_dir); + defer store.deinit(); + + const summary = try demo_seed.seed(allocator, &store); + try writeJsonToStdout(allocator, summary); +} + fn parseServeArgs(allocator: std.mem.Allocator, args: *ArgCursor) !struct { runtime: RuntimeConfig } { var overrides = RuntimeOverrides{}; @@ -598,6 +614,7 @@ fn printUsage() void { \\ nullwatch evals [--run-id ID] [--verdict VERDICT] [--eval-key KEY] [--scorer NAME] [--dataset NAME] [--limit N] \\ nullwatch ingest-span --json '' [--data-dir PATH] [--config PATH] \\ nullwatch ingest-eval --json '' [--data-dir PATH] [--config PATH] + \\ nullwatch demo-seed [--data-dir PATH] [--config PATH] \\ nullwatch --export-manifest \\ nullwatch --from-json '' \\ nullwatch version @@ -625,6 +642,7 @@ fn printUsage() void { test { _ = api; _ = config; + _ = demo_seed; _ = domain; _ = Store; _ = @import("export_manifest.zig"); diff --git a/tests/test_e2e.sh b/tests/test_e2e.sh index 7f6a202..476086c 100755 --- a/tests/test_e2e.sh +++ b/tests/test_e2e.sh @@ -80,4 +80,18 @@ NULLWATCH_HOME="${HOME_DIR}" zig build run -- summary | grep -q '"span_count": 2 NULLWATCH_HOME="${HOME_DIR}" zig build run -- runs --verdict pass | grep -q '"run_id": "run-123"' NULLWATCH_HOME="${HOME_DIR}" zig build run -- spans --tool-name shell | grep -q '"run_id": "run-otlp"' +DEMO_DATA_DIR="${TMP_DIR}/demo-data" +DEMO_FIRST="$(cd "${ROOT_DIR}" && zig build run -- demo-seed --data-dir "${DEMO_DATA_DIR}")" +echo "${DEMO_FIRST}" | grep -q '"runs_created": 3' +echo "${DEMO_FIRST}" | grep -q '"runs_skipped": 0' +echo "${DEMO_FIRST}" | grep -q '"spans_created": 11' +echo "${DEMO_FIRST}" | grep -q '"evals_created": 3' + +DEMO_SECOND="$(cd "${ROOT_DIR}" && zig build run -- demo-seed --data-dir "${DEMO_DATA_DIR}")" +echo "${DEMO_SECOND}" | grep -q '"runs_created": 0' +echo "${DEMO_SECOND}" | grep -q '"runs_skipped": 3' + +zig build run -- summary --data-dir "${DEMO_DATA_DIR}" | grep -q '"run_count": 3' +zig build run -- run demo-tool-failure --data-dir "${DEMO_DATA_DIR}" | grep -q '"overall_verdict": "fail"' + echo "nullwatch e2e: ok"