From ba0228ccf8d1dc126932af58ecbd5dc66af7925e Mon Sep 17 00:00:00 2001 From: Earl Cameron Date: Sun, 26 Oct 2025 13:58:14 -0700 Subject: [PATCH 1/4] chore: stage reactor async queue support --- README.md | 14 + build.zig | 145 ++++ config.json | 16 + docs/PHASE2_REACTOR_PLAN.md | 160 +++++ src/features/blog/effects.zig | 109 ++- src/features/todos/effects.zig | 2 +- src/zerver/bootstrap/init.zig | 14 +- src/zerver/core/types.zig | 152 ++++ src/zerver/impure/executor.zig | 767 ++++++++++++++++++++- src/zerver/impure/server.zig | 2 +- src/zerver/observability/slog.zig | 17 + src/zerver/root.zig | 6 + src/zerver/runtime/config.zig | 176 +++++ src/zerver/runtime/global.zig | 4 + src/zerver/runtime/listener.zig | 2 +- src/zerver/runtime/reactor/effectors.zig | 260 +++++++ src/zerver/runtime/reactor/job_system.zig | 319 +++++++++ src/zerver/runtime/reactor/join.zig | 114 +++ src/zerver/runtime/reactor/libuv.zig | 53 ++ src/zerver/runtime/reactor/saga.zig | 30 + src/zerver/runtime/reactor/task_system.zig | 145 ++++ src/zerver/runtime/resources.zig | 155 ++++- tests/libuv_smoke.zig | 195 ++++++ tests/unit/reactor_effectors.zig | 69 ++ tests/unit/reactor_job_system.zig | 56 ++ tests/unit/reactor_join.zig | 81 +++ tests/unit/reactor_saga.zig | 24 + tests/unit/reactor_task_system.zig | 97 +++ third_party/libuv | 1 + 29 files changed, 3135 insertions(+), 50 deletions(-) create mode 100644 docs/PHASE2_REACTOR_PLAN.md create mode 100644 src/zerver/runtime/reactor/effectors.zig create mode 100644 src/zerver/runtime/reactor/job_system.zig create mode 100644 src/zerver/runtime/reactor/join.zig create mode 100644 src/zerver/runtime/reactor/libuv.zig create mode 100644 src/zerver/runtime/reactor/saga.zig create mode 100644 src/zerver/runtime/reactor/task_system.zig create mode 100644 tests/libuv_smoke.zig create mode 100644 tests/unit/reactor_effectors.zig create mode 100644 tests/unit/reactor_job_system.zig create mode 100644 tests/unit/reactor_join.zig create mode 100644 tests/unit/reactor_saga.zig create mode 100644 tests/unit/reactor_task_system.zig create mode 160000 third_party/libuv diff --git a/README.md b/README.md index 4d26988..91e9fa1 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,20 @@ We welcome discussion, feedback, and contributions on the design. Please review --- +## Configuration + +Runtime settings live in `config.json`. The file now centralises: + +- database driver, file path, pool size, and busy timeout +- blocking thread pool sizing for legacy/middleware work +- reactor pools (continuation workers, effector workers, compute pool type/size) +- HTTP server bind address +- observability endpoints and Tempo auto-detection + +Tuning pool sizes here avoids recompilation; `runtime/config.zig` loads the `reactor` section and feeds those values into the job and task systems during startup. + +--- + ## Full Example: Todo CRUD API For a complete, runnable example demonstrating all of Zerver's core features working together, see [`examples/todo_crud.zig`](examples/todo_crud.zig). diff --git a/build.zig b/build.zig index eb8e4eb..bafe32b 100644 --- a/build.zig +++ b/build.zig @@ -1,5 +1,71 @@ const std = @import("std"); +const libuv_source_files = [_][]const u8{ + "third_party/libuv/src/fs-poll.c", + "third_party/libuv/src/idna.c", + "third_party/libuv/src/inet.c", + "third_party/libuv/src/random.c", + "third_party/libuv/src/strscpy.c", + "third_party/libuv/src/strtok.c", + "third_party/libuv/src/thread-common.c", + "third_party/libuv/src/threadpool.c", + "third_party/libuv/src/timer.c", + "third_party/libuv/src/uv-common.c", + "third_party/libuv/src/uv-data-getter-setters.c", + "third_party/libuv/src/version.c", + "third_party/libuv/src/win/async.c", + "third_party/libuv/src/win/core.c", + "third_party/libuv/src/win/detect-wakeup.c", + "third_party/libuv/src/win/dl.c", + "third_party/libuv/src/win/error.c", + "third_party/libuv/src/win/fs.c", + "third_party/libuv/src/win/fs-event.c", + "third_party/libuv/src/win/getaddrinfo.c", + "third_party/libuv/src/win/getnameinfo.c", + "third_party/libuv/src/win/handle.c", + "third_party/libuv/src/win/loop-watcher.c", + "third_party/libuv/src/win/pipe.c", + "third_party/libuv/src/win/thread.c", + "third_party/libuv/src/win/poll.c", + "third_party/libuv/src/win/process.c", + "third_party/libuv/src/win/process-stdio.c", + "third_party/libuv/src/win/signal.c", + "third_party/libuv/src/win/snprintf.c", + "third_party/libuv/src/win/stream.c", + "third_party/libuv/src/win/tcp.c", + "third_party/libuv/src/win/tty.c", + "third_party/libuv/src/win/udp.c", + "third_party/libuv/src/win/util.c", + "third_party/libuv/src/win/winapi.c", + "third_party/libuv/src/win/winsock.c", +}; + +const libuv_system_libs = [_][]const u8{ + "psapi", + "user32", + "advapi32", + "iphlpapi", + "userenv", + "ws2_32", + "dbghelp", + "ole32", + "shell32", +}; + +fn addLibuv(b: *std.Build, artifact: *std.Build.Step.Compile) void { + artifact.root_module.addIncludePath(b.path("third_party/libuv/include")); + artifact.root_module.addIncludePath(b.path("third_party/libuv/src")); + inline for (libuv_source_files) |path| { + artifact.addCSourceFile(.{ .file = b.path(path), .flags = &[_][]const u8{} }); + } + artifact.root_module.addCMacro("WIN32_LEAN_AND_MEAN", "1"); + artifact.root_module.addCMacro("_WIN32_WINNT", "0x0A00"); + artifact.root_module.addCMacro("_CRT_DECLARE_NONSTDC_NAMES", "0"); + inline for (libuv_system_libs) |name| { + artifact.linkSystemLibrary(name); + } +} + pub fn build(b: *std.Build) void { // Check Zig version compatibility (build-time check) comptime { @@ -32,6 +98,7 @@ pub fn build(b: *std.Build) void { }, }); exe.linkLibC(); + addLibuv(b, exe); const run_cmd = b.addRunArtifact(exe); run_cmd.step.dependOn(b.getInstallStep()); @@ -44,6 +111,11 @@ pub fn build(b: *std.Build) void { const zerver_mod = b.createModule(.{ .root_source_file = b.path("src/zerver/root.zig"), }); + zerver_mod.addIncludePath(b.path("third_party/libuv/include")); + zerver_mod.addIncludePath(b.path("third_party/libuv/src")); + zerver_mod.addCMacro("WIN32_LEAN_AND_MEAN", "1"); + zerver_mod.addCMacro("_WIN32_WINNT", "0x0A00"); + zerver_mod.addCMacro("_CRT_DECLARE_NONSTDC_NAMES", "0"); // Development helper steps const test_step = b.step("test", "Run all tests"); @@ -67,6 +139,79 @@ pub fn build(b: *std.Build) void { test_exe.linkLibC(); test_step.dependOn(&b.addRunArtifact(test_exe).step); + const libuv_smoke = b.addExecutable(.{ + .name = "libuv_smoke", + .root_module = b.createModule(.{ + .root_source_file = b.path("tests/libuv_smoke.zig"), + .target = target, + .optimize = optimize, + }), + }); + libuv_smoke.linkLibC(); + addLibuv(b, libuv_smoke); + const libuv_smoke_run = b.addRunArtifact(libuv_smoke); + test_step.dependOn(&libuv_smoke_run.step); + const libuv_smoke_step = b.step("libuv_smoke", "Run the libuv smoke test"); + libuv_smoke_step.dependOn(&libuv_smoke_run.step); + + const join_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("tests/unit/reactor_join.zig"), + .target = target, + .optimize = optimize, + }), + }); + join_tests.root_module.addImport("zerver", zerver_mod); + const join_tests_run = b.addRunArtifact(join_tests); + const reactor_tests_step = b.step("reactor_tests", "Run reactor unit tests"); + reactor_tests_step.dependOn(&join_tests_run.step); + + const job_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("tests/unit/reactor_job_system.zig"), + .target = target, + .optimize = optimize, + }), + }); + job_tests.root_module.addImport("zerver", zerver_mod); + const job_tests_run = b.addRunArtifact(job_tests); + reactor_tests_step.dependOn(&job_tests_run.step); + + const effectors_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("tests/unit/reactor_effectors.zig"), + .target = target, + .optimize = optimize, + }), + }); + effectors_tests.root_module.addImport("zerver", zerver_mod); + effectors_tests.linkLibC(); + addLibuv(b, effectors_tests); + const effectors_tests_run = b.addRunArtifact(effectors_tests); + reactor_tests_step.dependOn(&effectors_tests_run.step); + + const saga_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("tests/unit/reactor_saga.zig"), + .target = target, + .optimize = optimize, + }), + }); + saga_tests.root_module.addImport("zerver", zerver_mod); + const saga_tests_run = b.addRunArtifact(saga_tests); + reactor_tests_step.dependOn(&saga_tests_run.step); + + const task_system_tests = b.addTest(.{ + .root_module = b.createModule(.{ + .root_source_file = b.path("tests/unit/reactor_task_system.zig"), + .target = target, + .optimize = optimize, + }), + }); + task_system_tests.root_module.addImport("zerver", zerver_mod); + const task_system_tests_run = b.addRunArtifact(task_system_tests); + reactor_tests_step.dependOn(&task_system_tests_run.step); + const fmt_step = b.step("fmt", "Format all Zig files"); const fmt_cmd = b.addSystemCommand(&[_][]const u8{ "zig", "fmt", "." }); fmt_step.dependOn(&fmt_cmd.step); diff --git a/config.json b/config.json index e8d3663..1200ade 100644 --- a/config.json +++ b/config.json @@ -8,6 +8,22 @@ "thread_pool": { "worker_count": 4 }, + "reactor": { + "enabled": true, + "continuation_pool": { + "size": 4, + "queue_capacity": 1024 + }, + "effector_pool": { + "size": 4, + "queue_capacity": 1024 + }, + "compute_pool": { + "type": "disabled", + "size": 0, + "queue_capacity": 0 + } + }, "server": { "host": "127.0.0.1", "port": 8080 diff --git a/docs/PHASE2_REACTOR_PLAN.md b/docs/PHASE2_REACTOR_PLAN.md new file mode 100644 index 0000000..b22ee82 --- /dev/null +++ b/docs/PHASE2_REACTOR_PLAN.md @@ -0,0 +1,160 @@ +# Phase 2 Reactor & Scheduler Plan + +Owner: TBD +Status: Draft (2025-10-26) + +## 1. Objectives + +- **Spec alignment**: Implement the Phase-2 proactor + scheduler upgrade described in `docs/SPEC.md` §10 while keeping the public API (`CtxView`, `Decision`, `Effect`) stable (§13). +- **Transparent engine swap**: Preserve the blocking MVP semantics at the API boundary; `Server.listen` remains synchronous while work is delegated to libuv-backed workers. +- **Deterministic orchestration**: Honour `Decision.Need` join contracts (§4.2) with accurate bookkeeping, ensuring required/optional semantics, join modes, and continuation scheduling are correct under concurrency. +- **Graceful lifecycle**: Provide clean startup/shutdown, cancellation, and backpressure mechanisms consistent with §10.2 and §4.1 (cleanup, `onExit`). +- **Trace-first runtime**: Emit structured events (spec §8) for loop operations, effect dispatch, completion, and continuations without imposing excessive overhead. + +## 2. Non-Goals + +- Shipping production-grade HTTP/DB drivers: we will stub or adapt existing MVP blocking code for now. +- Implementing OTLP export, circuit breakers, or retry budgets in this iteration; we focus on the core reactor and scheduler scaffolding. +- Replacing all MVP execution paths immediately. We target an opt-in path behind a build flag/env toggle, then iterate. + +## 3. Architecture Overview + +```text +[app steps] ──(Decision)──▶ [Interpreter] + │ + ▼ + [Scheduler Queue] + │ + ┌──────────────────┴──────────────────┐ + ▼ ▼ + [CPU Worker Pool] [Effect Dispatcher] + │ │ + ▼ ▼ + run continuations libuv loop (uv_run) + │ + ┌─────────────────────────┴─────────────────────────┐ + ▼ ▼ ▼ + Timers (uv_timer_t) Async (uv_async_t) Thread pool (uv_queue_work) +``` + +**Interpreter**: Translates `Decision.Need` into reactor tasks, creates join state (counters, required flags, slot tokens), and enqueues continuation work when joins resolve. + +**Scheduler Queue**: Lock-free multi-producer/multi-consumer structure (likely Zig `std.atomic.Queue` or custom) feeding CPU workers with ready continuations. + +**Effect Dispatcher**: Wraps libuv handles and thread-pool submissions, ensuring each effect’s timeout, retry policy, and optional vs required status is tracked. + +**Join State**: A per-request/per-Need structure containing: +- outstanding count +- required failure flag +- optional failures log (for traces) +- resume function pointer + context pointer (continuation slot) +- Mode/Join semantics from `Decision.Need` + +## 4. Work Streams & Milestones + +### 4.1 Build & Tooling + +1. **Build helper**: Encapsulate libuv source list and macros in `build.zig` helper for reuse across binaries/tests. +2. **CI sanity**: Ensure `zig build libuv_smoke` runs in automation (derive from new plan). Add optional `zig test` gating once interpreter integration lands. + +### 4.2 Runtime Foundations + +1. **`runtime/reactor/libuv.zig`** + - Wrap libuv loop creation, `uv_async_t`, `uv_timer_t`, and thread-pool submission. + - Provide RAII-style helpers (`init`, `deinit`, `closeHandle`), returning Zig errors mapped to `LibuvError` equivalents. + +2. **`runtime/reactor/join.zig`** + - Define join state struct with atomic counters and completion callbacks. + - Implement helpers: `initJoin`, `registerEffect`, `markSuccess`, `markFailure`, `shouldResume`. + +3. **`runtime/reactor/effect_dispatch.zig`** + - Translate `Effect` union into specific libuv operations. + - Enforce `timeout_ms`, `retry`, and required vs optional semantics. + - Surface completions back into join state via scheduler queue. + +4. **`runtime/reactor/task_system.zig`** + - Build on shared `JobSystem` to provide continuation vs compute queues with shared shutdown semantics. + - Expose helper accessors so effect dispatchers and continuations can share pools safely. + - Source pool sizes and compute mode from `config.json` (`reactor` section) so deployments can tune queues without recompiling. + +5. **`core/types.zig` updates** + - Extend `Effect` union to cover all HTTP verbs for dispatcher parity. + - Add `Need.compensations` metadata hook; saga execution remains stubbed but can be threaded through pipelines early. + +6. **`runtime/scheduler.zig`** + - Manage CPU worker threads (likely Zig threads) that process continuations. + - Provide APIs: `spawn`, `enqueue(ContinuationTask)`, `shutdown`. + +7. **`runtime/runtime_engine.zig`** + - Orchestrate interpreter ↔ reactor boundaries. + - Accept MVP interpreter callbacks but route through new scheduler when enabled. + +### 4.3 Integrations + +1. **Interpreter bridge**: Update the existing step execution pipeline to call the scheduler when `Decision.Need` appears. +2. **Slot writes**: Ensure effect completions write tokens via `CtxBase._put` before enqueuing continuations. +3. **Error propagation**: Required effect failures synthesize `Decision.Fail` with `Error` per spec §7. +4. **Shutdown**: Connect `Server.deinit` / `listen` teardown to reactor shutdown (cancel outstanding work, flush queues). + +### 4.4 Observability & Testing + +1. **Tracing hooks**: Mirror spec §8 events: `ReactorEffectScheduled`, `ReactorEffectCompleted`, `JoinSatisfied`, `ContinuationEnqueued`. +2. **Smoke suite expansion**: + - Multi-effect join cases (all / any / first_success). + - Timeout + cancellation scenario (close loop before completion, ensure join handles gracefully). + - Retry policy enforcement (simulate failure; ensure retry budget decrements). + +3. **Integration tests**: Extend `tests/integration/server_test.zig` (or add new) to run a real route via libuv engine path. +4. **Bench harness** (optional): Basic throughput benchmark comparing MVP vs reactor path. + +### 4.5 Compute Worker Pool + +1. **Dedicated CPU queue**: Introduce a separate queue for long-running, CPU-bound tasks (analytics, JSON encoding, compression) so they do not block continuation workers. +2. **Worker sizing**: Expose configuration for compute worker count (default `max(1, cpu_count / 2)`) and allow runtime overrides via `Server.Config` extensions. +3. **Task API**: Provide `scheduleCompute(work: *const fn(*CtxBase) void, ctx: *CtxBase)` or similar so steps can explicitly offload heavy work while guaranteeing arena ownership rules. +4. **Tracing hooks**: Emit `ComputeTaskScheduled`, `ComputeTaskStarted`, `ComputeTaskCompleted` events feeding into the tracing pipeline for observability. +5. **Backpressure**: Enforce bounded compute queue size (configurable). When full, surface a `Decision.Fail` with `Error{kind = TooManyRequests}` or fallback strategy per spec §10.2. +6. **Testing**: Add stress tests that enqueue many CPU tasks to ensure fairness between continuation workers and compute workers; confirm shutdown drains both pools cleanly. + +### 4.6 Saga & Compensation Stubs + +1. **`runtime/reactor/saga.zig`** + - Provide a minimal stub (`SagaLog`) returning `error.Unimplemented` so upper layers can begin wiring compensation metadata without committing to behaviour. +2. **Compensation pipeline** + - Thread `Need.compensations` through interpreter plumbing while keeping execution disabled. +3. **Documentation** + - Mark saga rollback work as deferred; capture open questions for ordering, retry strategy, and observability before implementing in a later phase. + +## 5. Incremental Delivery Plan + +| Step | Deliverable | Validation | +| --- | --- | --- | +| 1 | libuv build helper + smoke tests (done) | `zig build libuv_smoke` | +| 2 | Reactor wrapper module + unit tests | new tests under `tests/unit/reactor_*.zig` | +| 3 | Join state implementation | deterministic tests covering all join modes | +| 4 | Effect dispatcher (HTTP/DB placeholders) | fake effect completions + timeouts | +| 5 | Task system coordinating continuation/compute pools | unit tests under `tests/unit/reactor_task_system.zig` | +| 6 | Compensation stubs threaded through Need | unit coverage validating `error.Unimplemented` | +| 7 | Scheduler integration with continuations | existing ReqTest + new integration tests | +| 8 | Feature flag to switch runtime | CLI/env gating, docs updated | + +## 6. Open Questions + +- **Backpressure strategy**: Do we implement per-target caps now or stub metrics/coarse limits first? (Spec §10.2 mentions bounded queues.) +- **Cancellation semantics**: Should optional effects continue completing post-resume (MVP behaviour) or be cancelled immediately when join condition is met? (Spec Appendix B suggests optional completions may still occur in MVP.) +- **Retry policy location**: Implement inside dispatcher now or defer until dedicated policy manager is available? +- **Configuration surface**: Where do we expose thread counts, queue depths, and timeout defaults? (Candidate: extend `Server.Config`.) + +## 7. Documentation & Communication + +- Update `docs/SPEC.md` once implementation details differ or solidify (e.g., actual backpressure approach). +- Maintain changelog entries summarising the experimental reactor work. +- Add developer guide in `docs/` for enabling the Phase 2 engine (`docs/PHASE2_USAGE.md`, future work). + +--- + +**Next Immediate Actions** + +1. Factor libuv build helper (`libuv_sources` handling) into reusable function in `build.zig`. +2. Scaffold `src/zerver/runtime/reactor/libuv.zig` with loop setup/teardown and a placeholder API. +3. Expand smoke tests to cover join behaviours as soon as join module exists. diff --git a/src/features/blog/effects.zig b/src/features/blog/effects.zig index 5b3f58a..56099cd 100644 --- a/src/features/blog/effects.zig +++ b/src/features/blog/effects.zig @@ -1,11 +1,14 @@ const std = @import("std"); const zerver = @import("../../zerver/root.zig"); const slog = @import("../../zerver/observability/slog.zig"); +const effectors = @import("../../zerver/runtime/reactor/effectors.zig"); const schema = @import("schema.zig"); const sql = @import("../../zerver/sql/mod.zig"); const runtime_resources = @import("../../zerver/runtime/resources.zig"); const runtime_global = @import("../../zerver/runtime/global.zig"); +const types = zerver.types; + const allocator = std.heap.c_allocator; const ValueConvertError = error{UnexpectedType}; @@ -16,7 +19,10 @@ pub fn initialize(resources: *runtime_resources.RuntimeResources) !void { schema_mutex.lock(); defer schema_mutex.unlock(); - if (schema_initialized) return; + if (schema_initialized) { + registerReactorHandlers(resources); + return; + } var lease = try resources.acquireConnection(); defer lease.release(); @@ -24,9 +30,11 @@ pub fn initialize(resources: *runtime_resources.RuntimeResources) !void { try schema.initSchema(lease.connection()); schema_initialized = true; slog.info("blog sqlite schema ensured", &.{}); + + registerReactorHandlers(resources); } -pub fn effectHandler(effect: *const zerver.Effect, _timeout_ms: u32) anyerror!zerver.executor.EffectResult { +pub fn effectHandler(effect: *const zerver.Effect, _timeout_ms: u32) anyerror!zerver.types.EffectResult { _ = _timeout_ms; const resources = runtime_global.get(); @@ -95,7 +103,90 @@ pub fn effectHandler(effect: *const zerver.Effect, _timeout_ms: u32) anyerror!ze } } -fn handleDbGet(database: *sql.db.Connection, key: []const u8) !zerver.executor.EffectResult { +fn registerReactorHandlers(resources: *runtime_resources.RuntimeResources) void { + if (resources.reactorEffectDispatcher()) |dispatcher| { + dispatcher.setDbGetHandler(reactorDbGetHandler); + dispatcher.setDbPutHandler(reactorDbPutHandler); + dispatcher.setDbDelHandler(reactorDbDelHandler); + } +} + +fn reactorDbGetHandler(_: *effectors.Context, payload: types.DbGet) effectors.DispatchError!types.EffectResult { + slog.debug("blog reactor db_get", &.{ + slog.Attr.string("key", payload.key), + slog.Attr.uint("token", payload.token), + }); + + const resources = runtime_global.get(); + var lease = resources.acquireConnection() catch |err| { + slog.err("blog reactor db acquire failed", &.{ + slog.Attr.string("error", @errorName(err)), + }); + return .{ .failure = unexpectedError("db_acquire") }; + }; + defer lease.release(); + + const conn = lease.connection(); + return handleDbGet(conn, payload.key) catch |err| { + slog.err("blog reactor db_get error", &.{ + slog.Attr.string("key", payload.key), + slog.Attr.string("error", @errorName(err)), + }); + return .{ .failure = unexpectedError("db_get") }; + }; +} + +fn reactorDbPutHandler(_: *effectors.Context, payload: types.DbPut) effectors.DispatchError!types.EffectResult { + slog.debug("blog reactor db_put", &.{ + slog.Attr.string("key", payload.key), + slog.Attr.uint("token", payload.token), + }); + + const resources = runtime_global.get(); + var lease = resources.acquireConnection() catch |err| { + slog.err("blog reactor db acquire failed", &.{ + slog.Attr.string("error", @errorName(err)), + }); + return .{ .failure = unexpectedError("db_acquire") }; + }; + defer lease.release(); + + const conn = lease.connection(); + return handleDbPut(conn, payload.key, payload.value) catch |err| { + slog.err("blog reactor db_put error", &.{ + slog.Attr.string("key", payload.key), + slog.Attr.string("error", @errorName(err)), + }); + return .{ .failure = unexpectedError("db_put") }; + }; +} + +fn reactorDbDelHandler(_: *effectors.Context, payload: types.DbDel) effectors.DispatchError!types.EffectResult { + slog.debug("blog reactor db_del", &.{ + slog.Attr.string("key", payload.key), + slog.Attr.uint("token", payload.token), + }); + + const resources = runtime_global.get(); + var lease = resources.acquireConnection() catch |err| { + slog.err("blog reactor db acquire failed", &.{ + slog.Attr.string("error", @errorName(err)), + }); + return .{ .failure = unexpectedError("db_acquire") }; + }; + defer lease.release(); + + const conn = lease.connection(); + return handleDbDel(conn, payload.key) catch |err| { + slog.err("blog reactor db_del error", &.{ + slog.Attr.string("key", payload.key), + slog.Attr.string("error", @errorName(err)), + }); + return .{ .failure = unexpectedError("db_del") }; + }; +} + +fn handleDbGet(database: *sql.db.Connection, key: []const u8) !zerver.types.EffectResult { if (std.mem.eql(u8, key, "posts")) { return getAllPosts(database); } else if (std.mem.startsWith(u8, key, "posts/")) { @@ -112,7 +203,7 @@ fn handleDbGet(database: *sql.db.Connection, key: []const u8) !zerver.executor.E } }; } -fn handleDbPut(database: *sql.db.Connection, key: []const u8, value: []const u8) !zerver.executor.EffectResult { +fn handleDbPut(database: *sql.db.Connection, key: []const u8, value: []const u8) !zerver.types.EffectResult { if (std.mem.startsWith(u8, key, "posts/")) { try putPost(database, value); const empty_ptr = @constCast(&[_]u8{}); @@ -129,7 +220,7 @@ fn handleDbPut(database: *sql.db.Connection, key: []const u8, value: []const u8) } }; } -fn handleDbDel(database: *sql.db.Connection, key: []const u8) !zerver.executor.EffectResult { +fn handleDbDel(database: *sql.db.Connection, key: []const u8) !zerver.types.EffectResult { if (std.mem.startsWith(u8, key, "posts/")) { try deletePost(database, key[6..]); const empty_ptr = @constCast(&[_]u8{}); @@ -146,7 +237,7 @@ fn handleDbDel(database: *sql.db.Connection, key: []const u8) !zerver.executor.E } }; } -fn getAllPosts(database: *sql.db.Connection) !zerver.executor.EffectResult { +fn getAllPosts(database: *sql.db.Connection) !zerver.types.EffectResult { var stmt = try database.prepare("SELECT id, title, content, author, created_at, updated_at FROM posts ORDER BY created_at DESC"); defer stmt.deinit(); @@ -172,7 +263,7 @@ fn getAllPosts(database: *sql.db.Connection) !zerver.executor.EffectResult { return .{ .success = .{ .bytes = data, .allocator = allocator } }; } -fn getPost(database: *sql.db.Connection, id: []const u8) !zerver.executor.EffectResult { +fn getPost(database: *sql.db.Connection, id: []const u8) !zerver.types.EffectResult { var stmt = try database.prepare("SELECT id, title, content, author, created_at, updated_at FROM posts WHERE id = ?"); defer stmt.deinit(); @@ -196,7 +287,7 @@ fn getPost(database: *sql.db.Connection, id: []const u8) !zerver.executor.Effect } }; } -fn getCommentsForPost(database: *sql.db.Connection, post_id: []const u8) !zerver.executor.EffectResult { +fn getCommentsForPost(database: *sql.db.Connection, post_id: []const u8) !zerver.types.EffectResult { var stmt = try database.prepare("SELECT id, post_id, content, author, created_at FROM comments WHERE post_id = ? ORDER BY created_at ASC"); defer stmt.deinit(); @@ -224,7 +315,7 @@ fn getCommentsForPost(database: *sql.db.Connection, post_id: []const u8) !zerver return .{ .success = .{ .bytes = data, .allocator = allocator } }; } -fn getComment(database: *sql.db.Connection, id: []const u8) !zerver.executor.EffectResult { +fn getComment(database: *sql.db.Connection, id: []const u8) !zerver.types.EffectResult { var stmt = try database.prepare("SELECT id, post_id, content, author, created_at FROM comments WHERE id = ?"); defer stmt.deinit(); diff --git a/src/features/todos/effects.zig b/src/features/todos/effects.zig index 853f8b7..cef7111 100644 --- a/src/features/todos/effects.zig +++ b/src/features/todos/effects.zig @@ -4,7 +4,7 @@ const zerver = @import("../../zerver/root.zig"); const slog = @import("../../zerver/observability/slog.zig"); // Effect handler (mock database) -pub fn effectHandler(effect: *const zerver.Effect, _timeout_ms: u32) anyerror!zerver.executor.EffectResult { +pub fn effectHandler(effect: *const zerver.Effect, _timeout_ms: u32) anyerror!zerver.types.EffectResult { slog.debug("Processing database effect", &.{ slog.Attr.string("effect_type", @tagName(effect.*)), }); diff --git a/src/zerver/bootstrap/init.zig b/src/zerver/bootstrap/init.zig index bc78d90..dda7e10 100644 --- a/src/zerver/bootstrap/init.zig +++ b/src/zerver/bootstrap/init.zig @@ -21,7 +21,7 @@ const blog_effects = @import("../../features/blog/effects.zig"); const blog_errors = @import("../../features/blog/errors.zig"); /// Composite effect handler that routes to the appropriate feature handler -fn compositeEffectHandler(effect: *const root.Effect, timeout_ms: u32) anyerror!root.executor.EffectResult { +fn compositeEffectHandler(effect: *const root.Effect, timeout_ms: u32) anyerror!root.types.EffectResult { // Use blog effects handler return try blog_effects.effectHandler(effect, timeout_ms); } @@ -80,6 +80,18 @@ pub fn initializeServer(allocator: std.mem.Allocator) !Initialization { slog.Attr.int("port", @as(i64, @intCast(server_port))), }); + const reactor_cfg = app_config.reactor; + slog.info("reactor_config", &[_]slog.Attr{ + slog.Attr.@"bool"("enabled", reactor_cfg.enabled), + slog.Attr.uint("continuation_workers", reactor_cfg.continuation_pool.size), + slog.Attr.uint("continuation_queue", reactor_cfg.continuation_pool.queue_capacity), + slog.Attr.uint("effector_workers", reactor_cfg.effector_pool.size), + slog.Attr.uint("effector_queue", reactor_cfg.effector_pool.queue_capacity), + slog.Attr.string("compute_kind", @tagName(reactor_cfg.compute_pool.kind)), + slog.Attr.uint("compute_workers", reactor_cfg.compute_pool.size), + slog.Attr.uint("compute_queue", reactor_cfg.compute_pool.queue_capacity), + }); + if (app_config.observability.otlp_endpoint.len == 0) { if (try detectTempoEndpoint(allocator, &app_config.observability)) |detected_endpoint| { slog.info("tempo_auto_configured", &.{ diff --git a/src/zerver/core/types.zig b/src/zerver/core/types.zig index 59b5b85..f5e7358 100644 --- a/src/zerver/core/types.zig +++ b/src/zerver/core/types.zig @@ -140,6 +140,15 @@ pub const Error = struct { ctx: ErrorCtx, }; +/// Effect result: either success payload bytes or failure metadata. +pub const EffectResult = union(enum) { + success: struct { + bytes: []u8, + allocator: ?std.mem.Allocator, + }, + failure: Error, +}; + /// Retry policy with configurable parameters for fault tolerance. pub const Retry = struct { max: u8 = 0, // Maximum number of retries @@ -231,6 +240,79 @@ pub const HttpPost = struct { required: bool = true, }; +/// HTTP HEAD effect. +pub const HttpHead = struct { + url: []const u8, + headers: []const Header = &.{}, + token: u32, + timeout_ms: u32 = 1000, + retry: Retry = .{}, + required: bool = true, +}; + +/// HTTP PUT effect. +pub const HttpPut = struct { + url: []const u8, + body: []const u8, + headers: []const Header = &.{}, + token: u32, + timeout_ms: u32 = 1000, + retry: Retry = .{}, + required: bool = true, +}; + +/// HTTP DELETE effect. +pub const HttpDelete = struct { + url: []const u8, + body: []const u8 = "", + headers: []const Header = &.{}, + token: u32, + timeout_ms: u32 = 1000, + retry: Retry = .{}, + required: bool = true, +}; + +/// HTTP OPTIONS effect. +pub const HttpOptions = struct { + url: []const u8, + headers: []const Header = &.{}, + token: u32, + timeout_ms: u32 = 1000, + retry: Retry = .{}, + required: bool = true, +}; + +/// HTTP TRACE effect. +pub const HttpTrace = struct { + url: []const u8, + headers: []const Header = &.{}, + token: u32, + timeout_ms: u32 = 1000, + retry: Retry = .{}, + required: bool = true, +}; + +/// HTTP CONNECT effect. +pub const HttpConnect = struct { + url: []const u8, + headers: []const Header = &.{}, + token: u32, + timeout_ms: u32 = 1000, + retry: Retry = .{}, + required: bool = true, +}; + +/// HTTP PATCH effect. +pub const HttpPatch = struct { + url: []const u8, + body: []const u8, + headers: []const Header = &.{}, + token: u32, + timeout_ms: u32 = 1000, + retry: Retry = .{}, + required: bool = true, +}; + /// Database GET effect. pub const DbGet = struct { key: []const u8, @@ -285,16 +367,85 @@ pub const FileJsonWrite = struct { required: bool = true, }; +/// Compute-bound task scheduled on dedicated worker pool. +pub const ComputeTask = struct { + operation: []const u8, + token: u32, + timeout_ms: u32 = 0, + required: bool = true, + metadata: ?*const anyopaque = null, +}; + +/// Accelerator task (GPU/TPU/etc.) routed to specialized queue. +pub const AcceleratorTask = struct { + kernel: []const u8, + token: u32, + timeout_ms: u32 = 2000, + required: bool = true, + metadata: ?*const anyopaque = null, +}; + +/// Key-value cache read. +pub const KvCacheGet = struct { + key: []const u8, + token: u32, + timeout_ms: u32 = 50, + required: bool = true, +}; + +/// Key-value cache write. +pub const KvCacheSet = struct { + key: []const u8, + value: []const u8, + token: u32, + timeout_ms: u32 = 50, + required: bool = true, + ttl_ms: u32 = 0, +}; + +/// Key-value cache delete/invalidate. +pub const KvCacheDelete = struct { + key: []const u8, + token: u32, + timeout_ms: u32 = 50, + required: bool = false, +}; + /// An Effect represents a request to perform I/O (HTTP, DB, etc.). pub const Effect = union(enum) { http_get: HttpGet, + http_head: HttpHead, http_post: HttpPost, + http_put: HttpPut, + http_delete: HttpDelete, + http_options: HttpOptions, + http_trace: HttpTrace, + http_connect: HttpConnect, + http_patch: HttpPatch, db_get: DbGet, db_put: DbPut, db_del: DbDel, db_scan: DbScan, file_json_read: FileJsonRead, file_json_write: FileJsonWrite, + compute_task: ComputeTask, + accelerator_task: AcceleratorTask, + kv_cache_get: KvCacheGet, + kv_cache_set: KvCacheSet, + kv_cache_delete: KvCacheDelete, +}; + +/// Trigger condition for running compensating actions. +pub const CompensationTrigger = enum { + on_failure, + on_cancel, +}; + +/// Description of a compensating action for saga-style orchestration. +pub const Compensation = struct { + label: []const u8 = "", + trigger: CompensationTrigger = .on_failure, + effect: Effect, }; /// Mode for executing multiple effects. @@ -320,6 +471,7 @@ pub const Need = struct { mode: Mode, join: Join, continuation: ResumeFn, + compensations: []const Compensation = &.{}, }; pub const Decision = union(enum) { diff --git a/src/zerver/impure/executor.zig b/src/zerver/impure/executor.zig index 583579c..35982a2 100644 --- a/src/zerver/impure/executor.zig +++ b/src/zerver/impure/executor.zig @@ -13,36 +13,567 @@ const types = @import("../core/types.zig"); const ctx_module = @import("../core/ctx.zig"); const slog = @import("../observability/slog.zig"); const telemetry = @import("../observability/telemetry.zig"); +const runtime_global = @import("../runtime/global.zig"); +const effectors = @import("../runtime/reactor/effectors.zig"); +const reactor_join = @import("../runtime/reactor/join.zig"); +const reactor_jobs = @import("../runtime/reactor/job_system.zig"); +const reactor_task_system = @import("../runtime/reactor/task_system.zig"); + +pub const EffectResult = types.EffectResult; pub const ExecutionMode = enum { Synchronous, // Block on each effect Async, // Phase-2: async/await }; -/// Effect result: either success with data, or failure with error. -pub const EffectResult = union(enum) { - success: struct { - bytes: []u8, - allocator: ?std.mem.Allocator = null, - }, - failure: types.Error, // Failure details +const ReactorNeedRunner = struct { + const Completion = struct { + result: types.EffectResult, + required: bool, + effect: *const types.Effect, + sequence: usize, + }; + + const JobContext = struct { + runner: *ReactorNeedRunner, + effect: *const types.Effect, + timeout_ms: u32, + required: bool, + token: u32, + telemetry_sequence: usize, + }; + + allocator: std.mem.Allocator, + executor: *Executor, + ctx_base: *ctx_module.CtxBase, + need: types.Need, + depth: usize, + need_sequence: usize, + dispatcher: *effectors.EffectDispatcher, + effect_context: effectors.Context, + effector_jobs: *reactor_jobs.JobSystem, + task_system: ?*reactor_task_system.TaskSystem, + telemetry_ctx: ?*telemetry.Telemetry, + results: std.AutoHashMap(u32, Completion) = undefined, + mutex: std.Thread.Mutex = .{}, + cond: std.Thread.Condition = .{}, + outstanding: usize = 0, + completed: usize = 0, + failure_error: ?types.Error = null, + last_failure_error: ?types.Error = null, + insert_error: ?error{OutOfMemory} = null, + join_state: ?reactor_join.JoinState = null, + join_status: ?reactor_join.Status = null, + continuation_decision: ?types.Decision = null, + completed_continuation_ctx: ?*ContinuationJobContext = null, + + fn run(self: *ReactorNeedRunner) !types.Decision { + self.results = std.AutoHashMap(u32, Completion).init(self.allocator); + defer self.results.deinit(); + + self.outstanding = self.need.effects.len; + self.completed = 0; + self.failure_error = null; + self.last_failure_error = null; + self.insert_error = null; + self.join_status = null; + self.continuation_decision = null; + self.completed_continuation_ctx = null; + self.join_state = if (self.outstanding > 0) + reactor_join.JoinState.init(.{ + .mode = self.need.mode, + .join = self.need.join, + }, self.outstanding, countRequiredEffects(self.need.effects)) + else + null; + + slog.debug("reactor_need_start", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("effects", @as(u64, @intCast(self.outstanding))), + slog.Attr.uint("required", @as(u64, @intCast(countRequiredEffects(self.need.effects)))), + slog.Attr.string("mode", @tagName(self.need.mode)), + slog.Attr.string("join", @tagName(self.need.join)), + }); + + if (self.outstanding == 0) { + if (self.telemetry_ctx) |t| { + t.continuationResume(self.need_sequence, @intFromPtr(self.need.continuation), self.need.mode, self.need.join); + } + slog.debug("reactor_need_immediate_resume", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + }); + return self.executor.executeStepInternal(self.ctx_base, self.need.continuation, self.depth + 1); + } + + var index: usize = 0; + while (index < self.need.effects.len) : (index += 1) { + try self.scheduleEffect(&self.need.effects[index]); + } + + self.awaitCompletion(); + + slog.debug("reactor_need_completed", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("completed", @as(u64, @intCast(self.completed))), + }); + + if (self.insert_error) |err| return err; + const final_status = if (self.join_state != null) + self.join_status orelse reactor_join.Status.success + else + reactor_join.Status.success; + + if (final_status == .failure) { + slog.err("reactor_need_failure", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + }); + self.releaseResults(); + const failure = self.failure_error orelse self.last_failure_error orelse defaultJoinFailureError(); + return .{ .Fail = failure }; + } + + var iter = self.results.iterator(); + while (iter.next()) |entry| { + const token = entry.key_ptr.*; + const completion = entry.value_ptr.*; + + switch (completion.result) { + .success => |payload| { + const data = payload.bytes; + if (payload.allocator) |alloc| { + errdefer alloc.free(data); + } + try self.ctx_base.slotPutString(token, data); + if (payload.allocator) |alloc| { + alloc.free(data); + } + }, + .failure => |err| { + self.ctx_base.last_error = err; + }, + } + } + + if (self.telemetry_ctx) |t| { + t.continuationResume(self.need_sequence, @intFromPtr(self.need.continuation), self.need.mode, self.need.join); + } + + slog.debug("reactor_need_resume_ready", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("continuation", @as(u64, @intCast(@intFromPtr(self.need.continuation)))), + }); + + if (self.task_system) |ts| { + return try self.resumeContinuationViaTaskSystem(ts); + } + + return self.executor.executeStepInternal(self.ctx_base, self.need.continuation, self.depth + 1); + } + + fn scheduleEffect(self: *ReactorNeedRunner, effect_ptr: *const types.Effect) !void { + const timeout_ms = effectTimeout(effect_ptr.*); + const required = effectRequired(effect_ptr.*); + const token = effectToken(effect_ptr.*); + const target = effectTarget(effect_ptr.*); + const effect_sequence = if (self.telemetry_ctx) |t| + t.effectStart(.{ + .kind = @tagName(effect_ptr.*), + .token = token, + .required = required, + .mode = self.need.mode, + .join = self.need.join, + .timeout_ms = timeout_ms, + .target = target, + .need_sequence = self.need_sequence, + }) + else + 0; + + slog.debug("reactor_effect_schedule", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.string("effect", @tagName(effect_ptr.*)), + slog.Attr.uint("token", @as(u64, @intCast(token))), + slog.Attr.bool("required", required), + slog.Attr.uint("timeout_ms", @as(u64, timeout_ms)), + }); + + const job_ctx = try self.allocator.create(JobContext); + job_ctx.* = .{ + .runner = self, + .effect = effect_ptr, + .timeout_ms = timeout_ms, + .required = required, + .token = token, + .telemetry_sequence = effect_sequence, + }; + + const job = reactor_jobs.Job{ + .callback = reactorNeedJobCallback, + .ctx = @ptrCast(@alignCast(job_ctx)), + }; + + const submit_attempt = self.trySubmitCompute(effect_ptr.*, job) catch |submit_err| { + slog.err("reactor_effect_compute_submit_failed", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + slog.Attr.string("error", @errorName(submit_err)), + }); + self.allocator.destroy(job_ctx); + return submit_err; + }; + + if (submit_attempt) |submit_result| { + switch (submit_result) { + .done => { + slog.debug("reactor_effect_compute_enqueued", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + slog.Attr.uint("token", @as(u64, @intCast(token))), + }); + return; + }, + .fallback => { + slog.debug("reactor_effect_compute_fallback", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + slog.Attr.uint("token", @as(u64, @intCast(token))), + }); + }, + } + } + + self.effector_jobs.submit(job) catch |err| { + const failure = effectQueueFailure(effect_ptr.*, err); + slog.err("reactor_effect_enqueue_failed", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + slog.Attr.uint("token", @as(u64, @intCast(token))), + slog.Attr.string("error", @errorName(err)), + slog.Attr.string("queue", self.effector_jobs.label()), + }); + self.recordCompletion(job_ctx, .{ .failure = failure }); + self.allocator.destroy(job_ctx); + return; + }; + + slog.debug("reactor_effect_enqueued", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + slog.Attr.uint("token", @as(u64, @intCast(token))), + slog.Attr.string("queue", self.effector_jobs.label()), + }); + } + + fn executeEffect(self: *ReactorNeedRunner, effect_ptr: *const types.Effect, timeout_ms: u32) types.EffectResult { + slog.debug("reactor_effect_execute", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + slog.Attr.uint("token", @as(u64, @intCast(effectToken(effect_ptr.*)))), + slog.Attr.uint("timeout_ms", @as(u64, timeout_ms)), + }); + + const dispatch_result = blk: { + const res = self.dispatcher.dispatch(&self.effect_context, effect_ptr.*) catch |err| switch (err) { + error.UnsupportedEffect => { + slog.debug("reactor_effect_dispatch_unsupported", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + }); + break :blk null; + }, + }; + break :blk res; + }; + if (dispatch_result) |result| return result; + + return self.executor.effect_handler(effect_ptr, timeout_ms) catch { + const error_result: types.Error = .{ + .kind = types.ErrorCode.UpstreamUnavailable, + .ctx = .{ .what = "effect", .key = @tagName(effect_ptr.*) }, + }; + slog.err("reactor_effect_execute_failed", &.{ + slog.Attr.string("effect", @tagName(effect_ptr.*)), + }); + return .{ .failure = error_result }; + }; + } + + fn recordCompletion(self: *ReactorNeedRunner, job_ctx: *const JobContext, result: types.EffectResult) void { + var bytes_len: ?usize = null; + var error_ctx: ?types.ErrorCtx = null; + var failure_details: ?types.Error = null; + var is_success = false; + + switch (result) { + .success => |payload| { + is_success = true; + bytes_len = payload.bytes.len; + }, + .failure => |err| { + failure_details = err; + error_ctx = err.ctx; + }, + } + + const completed_bytes: u64 = if (bytes_len) |len| @intCast(len) else 0; + const error_key = if (error_ctx) |ctx| ctx.key else "unknown"; + slog.debug("reactor_effect_complete", &.{ + slog.Attr.string("effect", @tagName(job_ctx.effect.*)), + slog.Attr.uint("token", @as(u64, @intCast(job_ctx.token))), + slog.Attr.bool("success", is_success), + slog.Attr.uint("sequence", @as(u64, @intCast(job_ctx.telemetry_sequence))), + slog.Attr.bool("required", job_ctx.required), + slog.Attr.uint("bytes", completed_bytes), + slog.Attr.string("error", if (is_success) "" else error_key), + }); + + self.mutex.lock(); + defer self.mutex.unlock(); + + if (self.insert_error == null) { + self.results.put(job_ctx.token, .{ + .result = result, + .required = job_ctx.required, + .effect = job_ctx.effect, + .sequence = job_ctx.telemetry_sequence, + }) catch |err| { + self.insert_error = err; + }; + } + + if (!is_success and failure_details != null) { + self.last_failure_error = failure_details; + if (job_ctx.required) { + self.failure_error = failure_details; + } + } + + if (self.telemetry_ctx) |t| { + t.effectEnd(.{ + .sequence = job_ctx.telemetry_sequence, + .need_sequence = self.need_sequence, + .kind = @tagName(job_ctx.effect.*), + .token = job_ctx.token, + .required = job_ctx.required, + .success = is_success, + .bytes_len = bytes_len, + .error_ctx = error_ctx, + }); + } + + if (self.join_state) |*state| { + const resolution = state.record(.{ + .required = job_ctx.required, + .success = is_success, + }); + switch (resolution) { + .Pending => {}, + .Resume => |resume_info| { + self.join_status = resume_info.status; + }, + } + } + + self.completed += 1; + self.cond.signal(); + } + + fn awaitCompletion(self: *ReactorNeedRunner) void { + self.mutex.lock(); + defer self.mutex.unlock(); + + while (self.completed < self.outstanding) { + slog.debug("reactor_need_wait", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("completed", @as(u64, @intCast(self.completed))), + slog.Attr.uint("outstanding", @as(u64, @intCast(self.outstanding))), + }); + self.cond.wait(&self.mutex); + } + slog.debug("reactor_need_wake", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("completed", @as(u64, @intCast(self.completed))), + }); + } + + fn releaseResults(self: *ReactorNeedRunner) void { + var iter = self.results.iterator(); + while (iter.next()) |entry| { + switch (entry.value_ptr.result) { + .success => |payload| { + if (payload.allocator) |alloc| { + alloc.free(payload.bytes); + } + }, + .failure => {}, + } + } + } + + const SubmitComputeResult = enum { done, fallback }; + + fn trySubmitCompute(self: *ReactorNeedRunner, effect: types.Effect, job: reactor_jobs.Job) error{OutOfMemory}!?SubmitComputeResult { + if (!requiresComputePool(effect)) return null; + + const ts = self.task_system orelse return SubmitComputeResult.fallback; + + ts.submitCompute(job) catch |err| switch (err) { + error.NoComputePool, error.QueueFull, error.ShuttingDown => return SubmitComputeResult.fallback, + error.OutOfMemory => return error.OutOfMemory, + }; + + return SubmitComputeResult.done; + } + + const ContinuationJobContext = struct { + runner: *ReactorNeedRunner, + }; + + fn resumeContinuationViaTaskSystem(self: *ReactorNeedRunner, ts: *reactor_task_system.TaskSystem) !types.Decision { + const job_ctx = try self.allocator.create(ContinuationJobContext); + job_ctx.* = .{ .runner = self }; + + self.mutex.lock(); + self.continuation_decision = null; + self.mutex.unlock(); + + slog.debug("reactor_step_context_allocated", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), + }); + + const job = reactor_jobs.Job{ + .callback = continuationJobCallback, + .ctx = @ptrCast(@alignCast(job_ctx)), + }; + + slog.debug("reactor_step_schedule", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), + }); + + ts.submitContinuation(job) catch |err| { + self.allocator.destroy(job_ctx); + const failure = continuationQueueFailure(err); + slog.err("reactor_step_enqueue_failed", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.string("error", @errorName(err)), + }); + return .{ .Fail = failure }; + }; + + slog.debug("reactor_step_enqueued", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), + }); + + return self.waitForContinuationDecision(); + } + + fn waitForContinuationDecision(self: *ReactorNeedRunner) types.Decision { + self.mutex.lock(); + while (self.continuation_decision == null) { + slog.debug("reactor_step_wait", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + }); + self.cond.wait(&self.mutex); + } + + const decision = self.continuation_decision.?; + const job_ctx = self.completed_continuation_ctx; + self.continuation_decision = null; + self.completed_continuation_ctx = null; + self.mutex.unlock(); + + if (job_ctx) |ctx| { + slog.debug("reactor_step_context_free", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(ctx)))), + }); + self.allocator.destroy(ctx); + } + + slog.debug("reactor_step_decision", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.string("decision", @tagName(decision)), + }); + return decision; + } + + fn finishContinuation(self: *ReactorNeedRunner, decision: types.Decision) void { + self.mutex.lock(); + self.continuation_decision = decision; + self.mutex.unlock(); + self.cond.signal(); + slog.debug("reactor_step_publish", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.string("decision", @tagName(decision)), + }); + } + + fn markContinuationJobComplete(self: *ReactorNeedRunner, job_ctx: *ContinuationJobContext) void { + self.mutex.lock(); + defer self.mutex.unlock(); + slog.debug("reactor_step_context_complete", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), + }); + self.completed_continuation_ctx = job_ctx; + } }; +fn reactorNeedJobCallback(ctx_ptr: *anyopaque) void { + const job_ctx: *ReactorNeedRunner.JobContext = @ptrCast(@alignCast(ctx_ptr)); + const runner = job_ctx.runner; + slog.debug("reactor_effect_job_start", &.{ + slog.Attr.string("effect", @tagName(job_ctx.effect.*)), + slog.Attr.uint("token", @as(u64, @intCast(job_ctx.token))), + }); + const result = runner.executeEffect(job_ctx.effect, job_ctx.timeout_ms); + runner.recordCompletion(job_ctx, result); + slog.debug("reactor_effect_job_finish", &.{ + slog.Attr.string("effect", @tagName(job_ctx.effect.*)), + slog.Attr.uint("token", @as(u64, @intCast(job_ctx.token))), + }); + runner.allocator.destroy(job_ctx); +} + +fn continuationJobCallback(ctx_ptr: *anyopaque) void { + const job_ctx: *ReactorNeedRunner.ContinuationJobContext = @ptrCast(@alignCast(ctx_ptr)); + const runner = job_ctx.runner; + + slog.debug("reactor_step_job_start", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(runner.need_sequence))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), + }); + + const decision = runner.executor.executeStepInternal(runner.ctx_base, runner.need.continuation, runner.depth + 1) catch |err| { + const failure = failFromCrash(runner.executor, runner.ctx_base, "continuation", err, runner.depth + 1); + slog.err("reactor_step_job_crash", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(runner.need_sequence))), + slog.Attr.string("error", @errorName(err)), + }); + runner.markContinuationJobComplete(job_ctx); + runner.finishContinuation(failure); + return; + }; + + runner.markContinuationJobComplete(job_ctx); + runner.finishContinuation(decision); + slog.debug("reactor_step_job_finish", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(runner.need_sequence))), + slog.Attr.string("decision", @tagName(decision)), + }); +} + /// Executor manages step execution and effect handling. pub const Executor = struct { allocator: std.mem.Allocator, mode: ExecutionMode = .Synchronous, /// Effect handler: called to perform an effect and return result. - /// Signature: fn (*const Effect, timeout_ms: u32) anyerror!EffectResult - effect_handler: *const fn (*const types.Effect, u32) anyerror!EffectResult, + /// Signature: fn (*const Effect, timeout_ms: u32) anyerror!types.EffectResult + effect_handler: *const fn (*const types.Effect, u32) anyerror!types.EffectResult, /// Optional telemetry sink for tracing spans/events telemetry_ctx: ?*telemetry.Telemetry = null, pub fn init( allocator: std.mem.Allocator, - effect_handler: *const fn (*const types.Effect, u32) anyerror!EffectResult, + effect_handler: *const fn (*const types.Effect, u32) anyerror!types.EffectResult, ) Executor { return .{ .allocator = allocator, @@ -127,16 +658,30 @@ pub const Executor = struct { depth: usize, need_sequence: usize, ) anyerror!types.Decision { + if (try self.maybeExecuteNeedViaReactor(ctx_base, need, depth, need_sequence)) |reactor_decision| { + return reactor_decision; + } + // Track effect results by token (slot identifier) - var results = std.AutoHashMap(u32, EffectResult).init(ctx_base.allocator); + var results = std.AutoHashMap(u32, types.EffectResult).init(ctx_base.allocator); defer results.deinit(); - var had_required_failure = false; - var failure_error: ?types.Error = null; + const total_effects = need.effects.len; + const required_effects = countRequiredEffects(need.effects); + var join_state: ?reactor_join.JoinState = if (total_effects > 0) + reactor_join.JoinState.init(.{ + .mode = need.mode, + .join = need.join, + }, total_effects, required_effects) + else + null; + var join_status: ?reactor_join.Status = null; + + var required_failure: ?types.Error = null; + var last_failure: ?types.Error = null; // MVP: execute sequentially regardless of mode // Phase-2 can parallelize this - // TODO: Logical Error - The 'mode' (Parallel/Sequential) and 'join' strategies (all_required, any) are currently not fully respected due to sequential MVP execution. Revisit this logic when parallel execution is implemented to ensure correct behavior and avoid unintended side effects. for (need.effects) |effect| { const effect_kind = @tagName(effect); const token = effectToken(effect); @@ -177,9 +722,23 @@ pub const Executor = struct { }); } + last_failure = error_result; if (required) { - had_required_failure = true; - failure_error = error_result; + required_failure = error_result; + } + ctx_base.last_error = error_result; + + if (join_state) |*state| { + const resolution = state.record(.{ + .required = required, + .success = false, + }); + switch (resolution) { + .Pending => {}, + .Resume => |resume_info| { + join_status = resume_info.status; + }, + } } continue; }; @@ -215,29 +774,35 @@ pub const Executor = struct { } // If this is a required effect that failed, mark failure - if (required and !is_success) { - had_required_failure = true; - failure_error = failure_details; + if (!is_success and failure_details != null) { + last_failure = failure_details; + if (required) { + required_failure = failure_details; + } + ctx_base.last_error = failure_details.?; } - } - // Apply join strategy: decide when to resume - const should_resume = switch (need.join) { - .all => true, // always resume after all complete - .all_required => true, // MVP: same as all (Phase-2: can resume early) - .any => true, // MVP: same as all (Phase-2: would resume on first) - .first_success => !had_required_failure, // resume if any success or no required fails - // TODO: Bug - `.first_success` ignores whether any effect actually succeeded and never resumes early; it only checks the absence of required failures. - }; - - if (!should_resume) { - // Should not resume: required effect failed - return .{ .Fail = failure_error.? }; + if (join_state) |*state| { + const resolution = state.record(.{ + .required = required, + .success = is_success, + }); + switch (resolution) { + .Pending => {}, + .Resume => |resume_info| { + join_status = resume_info.status; + }, + } + } } - // If a required effect failed, fail the pipeline - if (had_required_failure) { - return .{ .Fail = failure_error.? }; + if (total_effects > 0) { + const final_status = join_status orelse reactor_join.Status.success; + if (final_status == .failure) { + releaseEffectResults(&results); + const failure = required_failure orelse last_failure orelse defaultJoinFailureError(); + return .{ .Fail = failure }; + } } // Store effect results in slots so steps can access them @@ -278,57 +843,187 @@ pub const Executor = struct { return self.executeStepInternal(ctx_base, need.continuation, depth + 1); } + + fn maybeExecuteNeedViaReactor( + self: *Executor, + ctx_base: *ctx_module.CtxBase, + need: types.Need, + depth: usize, + need_sequence: usize, + ) !?types.Decision { + const resources = runtime_global.maybeGet() orelse return null; + if (!resources.reactorEnabled()) return null; + + const dispatcher = resources.reactorEffectDispatcher() orelse return null; + const effect_context = resources.reactorEffectContext() orelse return null; + const effector_jobs = resources.reactorEffectorJobs() orelse return null; + + var runner = ReactorNeedRunner{ + .allocator = self.allocator, + .executor = self, + .ctx_base = ctx_base, + .need = need, + .depth = depth, + .need_sequence = need_sequence, + .dispatcher = dispatcher, + .effect_context = effect_context, + .effector_jobs = effector_jobs, + .task_system = resources.reactorTaskSystem(), + .telemetry_ctx = self.telemetry_ctx, + }; + + const decision = try runner.run(); + return decision; + } }; fn effectToken(effect: types.Effect) u32 { return switch (effect) { .http_get => |e| e.token, + .http_head => |e| e.token, .http_post => |e| e.token, + .http_put => |e| e.token, + .http_delete => |e| e.token, + .http_options => |e| e.token, + .http_trace => |e| e.token, + .http_connect => |e| e.token, + .http_patch => |e| e.token, .db_get => |e| e.token, .db_put => |e| e.token, .db_del => |e| e.token, .db_scan => |e| e.token, .file_json_read => |e| e.token, .file_json_write => |e| e.token, + .compute_task => |e| e.token, + .accelerator_task => |e| e.token, + .kv_cache_get => |e| e.token, + .kv_cache_set => |e| e.token, + .kv_cache_delete => |e| e.token, }; } fn effectTimeout(effect: types.Effect) u32 { return switch (effect) { .http_get => |e| e.timeout_ms, + .http_head => |e| e.timeout_ms, .http_post => |e| e.timeout_ms, + .http_put => |e| e.timeout_ms, + .http_delete => |e| e.timeout_ms, + .http_options => |e| e.timeout_ms, + .http_trace => |e| e.timeout_ms, + .http_connect => |e| e.timeout_ms, + .http_patch => |e| e.timeout_ms, .db_get => |e| e.timeout_ms, .db_put => |e| e.timeout_ms, .db_del => |e| e.timeout_ms, .db_scan => |e| e.timeout_ms, .file_json_read => 1000, .file_json_write => 1000, + .compute_task => |e| e.timeout_ms, + .accelerator_task => |e| e.timeout_ms, + .kv_cache_get => |e| e.timeout_ms, + .kv_cache_set => |e| e.timeout_ms, + .kv_cache_delete => |e| e.timeout_ms, }; } fn effectRequired(effect: types.Effect) bool { return switch (effect) { .http_get => |e| e.required, + .http_head => |e| e.required, .http_post => |e| e.required, + .http_put => |e| e.required, + .http_delete => |e| e.required, + .http_options => |e| e.required, + .http_trace => |e| e.required, + .http_connect => |e| e.required, + .http_patch => |e| e.required, .db_get => |e| e.required, .db_put => |e| e.required, .db_del => |e| e.required, .db_scan => |e| e.required, .file_json_read => |e| e.required, .file_json_write => |e| e.required, + .compute_task => |e| e.required, + .accelerator_task => |e| e.required, + .kv_cache_get => |e| e.required, + .kv_cache_set => |e| e.required, + .kv_cache_delete => |e| e.required, }; } fn effectTarget(effect: types.Effect) []const u8 { return switch (effect) { .http_get => |e| e.url, + .http_head => |e| e.url, .http_post => |e| e.url, + .http_put => |e| e.url, + .http_delete => |e| e.url, + .http_options => |e| e.url, + .http_trace => |e| e.url, + .http_connect => |e| e.url, + .http_patch => |e| e.url, .db_get => |e| e.key, .db_put => |e| e.key, .db_del => |e| e.key, .db_scan => |e| e.prefix, .file_json_read => |e| e.path, .file_json_write => |e| e.path, + .compute_task => |e| e.operation, + .accelerator_task => |e| e.kernel, + .kv_cache_get => |e| e.key, + .kv_cache_set => |e| e.key, + .kv_cache_delete => |e| e.key, + }; +} + +fn requiresComputePool(effect: types.Effect) bool { + return switch (effect) { + .compute_task, .accelerator_task => true, + else => false, + }; +} + +fn effectQueueFailure(effect: types.Effect, err: anyerror) types.Error { + return .{ + .kind = types.ErrorCode.UpstreamUnavailable, + .ctx = .{ .what = @tagName(effect), .key = @errorName(err) }, + }; +} + +fn continuationQueueFailure(err: anyerror) types.Error { + return .{ + .kind = types.ErrorCode.UpstreamUnavailable, + .ctx = .{ .what = "continuation", .key = @errorName(err) }, + }; +} + +fn countRequiredEffects(effects: []const types.Effect) usize { + var required: usize = 0; + for (effects) |effect| { + if (effectRequired(effect)) required += 1; + } + return required; +} + +fn releaseEffectResults(map: *std.AutoHashMap(u32, types.EffectResult)) void { + var iter = map.iterator(); + while (iter.next()) |entry| { + switch (entry.value_ptr.*) { + .success => |payload| { + if (payload.allocator) |alloc| { + alloc.free(payload.bytes); + } + }, + .failure => {}, + } + } +} + +fn defaultJoinFailureError() types.Error { + return .{ + .kind = types.ErrorCode.UpstreamUnavailable, + .ctx = .{ .what = "executor", .key = "join_failure" }, }; } @@ -360,7 +1055,7 @@ fn failFromCrash( /// Default effect handler that returns dummy results. /// Production systems would implement actual HTTP/DB clients. -pub fn defaultEffectHandler(_: *const types.Effect, _: u32) anyerror!EffectResult { +pub fn defaultEffectHandler(_: *const types.Effect, _: u32) anyerror!types.EffectResult { // MVP: return a dummy success result const empty: []u8 = &[_]u8{}; return .{ .success = .{ .bytes = empty, .allocator = null } }; diff --git a/src/zerver/impure/server.zig b/src/zerver/impure/server.zig index 7498ac5..8c72187 100644 --- a/src/zerver/impure/server.zig +++ b/src/zerver/impure/server.zig @@ -142,7 +142,7 @@ pub const Server = struct { pub fn init( allocator: std.mem.Allocator, cfg: Config, - effect_handler: *const fn (*const types.Effect, u32) anyerror!executor_module.EffectResult, + effect_handler: *const fn (*const types.Effect, u32) anyerror!types.EffectResult, ) !Server { return Server{ .allocator = allocator, diff --git a/src/zerver/observability/slog.zig b/src/zerver/observability/slog.zig index 3a151bf..6dbed4f 100644 --- a/src/zerver/observability/slog.zig +++ b/src/zerver/observability/slog.zig @@ -49,6 +49,7 @@ pub const Attr = struct { pub const Value = union(enum) { string: []const u8, + enum_tag: []const u8, int: i64, uint: u64, float: f64, @@ -67,6 +68,7 @@ pub const Attr = struct { switch (self) { .string => |s| try writer.writeAll(s), + .enum_tag => |s| try writer.writeAll(s), .int => |i| try writer.print("{}", .{i}), .uint => |u| try writer.print("{}", .{u}), .float => |f| try writer.print("{d}", .{f}), @@ -81,6 +83,18 @@ pub const Attr = struct { return .{ .key = key, .value = .{ .string = value } }; } + pub fn enumeration(key: []const u8, value: anytype) Attr { + const T = @TypeOf(value); + return switch (@typeInfo(T)) { + .Enum => .{ .key = key, .value = .{ .enum_tag = @tagName(value) } }, + .Union => |union_info| switch (union_info.tag_type) { + null => @compileError("Attr.enumeration expects a tagged union"), + else => .{ .key = key, .value = .{ .enum_tag = @tagName(value) } }, + }, + else => @compileError("Attr.enumeration expects an enum or tagged union"), + }; + } + pub fn int(key: []const u8, value: i64) Attr { return .{ .key = key, .value = .{ .int = value } }; } @@ -161,6 +175,9 @@ pub const TextHandler = struct { writer.writeAll(s) catch return; writer.writeByte('"') catch return; }, + .enum_tag => |s| { + writer.writeAll(s) catch return; + }, .int => |i| writer.print("{d}", .{i}) catch return, .uint => |u| writer.print("{d}", .{u}) catch return, .float => |f| writer.print("{d}", .{f}) catch return, diff --git a/src/zerver/root.zig b/src/zerver/root.zig index 10d3b94..4aeb279 100644 --- a/src/zerver/root.zig +++ b/src/zerver/root.zig @@ -14,6 +14,12 @@ pub const telemetry = @import("observability/telemetry.zig"); pub const otel = @import("observability/otel.zig"); pub const reqtest_module = @import("core/reqtest.zig"); pub const sql = @import("sql/mod.zig"); +pub const libuv_reactor = @import("runtime/reactor/libuv.zig"); +pub const reactor_join = @import("runtime/reactor/join.zig"); +pub const reactor_job_system = @import("runtime/reactor/job_system.zig"); +pub const reactor_effectors = @import("runtime/reactor/effectors.zig"); +pub const reactor_saga = @import("runtime/reactor/saga.zig"); +pub const reactor_task_system = @import("runtime/reactor/task_system.zig"); // Main types pub const CtxBase = ctx_module.CtxBase; diff --git a/src/zerver/runtime/config.zig b/src/zerver/runtime/config.zig index 63df2fb..9c6c100 100644 --- a/src/zerver/runtime/config.zig +++ b/src/zerver/runtime/config.zig @@ -4,6 +4,7 @@ const std = @import("std"); pub const AppConfig = struct { database: DatabaseConfig, thread_pool: ThreadPoolConfig = .{}, + reactor: ReactorConfig, observability: ObservabilityConfig, server: ServerConfig, @@ -27,6 +28,30 @@ pub const ThreadPoolConfig = struct { worker_count: usize = 1, }; +pub const ReactorPoolConfig = struct { + size: usize, + queue_capacity: usize, +}; + +pub const ComputePoolKind = enum { + disabled, + shared, + dedicated, +}; + +pub const ComputePoolConfig = struct { + kind: ComputePoolKind = .disabled, + size: usize = 0, + queue_capacity: usize = 0, +}; + +pub const ReactorConfig = struct { + enabled: bool, + continuation_pool: ReactorPoolConfig, + effector_pool: ReactorPoolConfig, + compute_pool: ComputePoolConfig, +}; + pub const ServerConfig = struct { host: []const u8 = "", port: u16 = 0, @@ -75,6 +100,24 @@ const RawThreadPoolConfig = struct { worker_count: ?usize = null, }; +const RawReactorPoolConfig = struct { + size: ?usize = null, + queue_capacity: ?usize = null, +}; + +const RawComputePoolConfig = struct { + type: ?[]const u8 = null, + size: ?usize = null, + queue_capacity: ?usize = null, +}; + +const RawReactorConfig = struct { + enabled: ?bool = null, + continuation_pool: ?RawReactorPoolConfig = null, + effector_pool: ?RawReactorPoolConfig = null, + compute_pool: ?RawComputePoolConfig = null, +}; + const RawServerConfig = struct { host: ?[]const u8 = null, port: ?u16 = null, @@ -98,6 +141,7 @@ const RawObservabilityConfig = struct { const RawAppConfig = struct { database: RawDatabaseConfig, thread_pool: ?RawThreadPoolConfig = null, + reactor: ?RawReactorConfig = null, observability: ?RawObservabilityConfig = null, server: ?RawServerConfig = null, }; @@ -132,6 +176,7 @@ pub fn load(allocator: std.mem.Allocator, path: []const u8) !AppConfig { errdefer observability.deinit(allocator); var server = try buildServerConfig(allocator, raw.server); errdefer server.deinit(allocator); + const reactor = try buildReactorConfig(raw.reactor, default_workers); return AppConfig{ .database = .{ @@ -146,11 +191,17 @@ pub fn load(allocator: std.mem.Allocator, path: []const u8) !AppConfig { else default_workers, }, + .reactor = reactor, .observability = observability, .server = server, }; } +const DEFAULT_REACTOR_ENABLED = false; +const DEFAULT_CONTINUATION_QUEUE_CAPACITY: usize = 1024; +const DEFAULT_EFFECTOR_QUEUE_CAPACITY: usize = 1024; +const DEFAULT_COMPUTE_QUEUE_CAPACITY: usize = 1024; + const DEFAULT_SERVICE_NAME = "zerver"; const DEFAULT_SERVICE_VERSION = "0.1.0"; const DEFAULT_ENVIRONMENT = "development"; @@ -164,6 +215,131 @@ const DEFAULT_OTLP_AUTODETECT_PORT: u16 = 4318; const DEFAULT_OTLP_AUTODETECT_PATH = "/v1/traces"; const DEFAULT_OTLP_AUTODETECT_SCHEME = "http"; +fn buildReactorConfig( + raw: ?RawReactorConfig, + default_workers: usize, +) !ReactorConfig { + const default_continuation_workers = if (default_workers == 0) 1 else default_workers; + const default_effector_workers = default_continuation_workers; + const default_compute_workers = if (default_continuation_workers <= 1) + 1 + else + (default_continuation_workers + 1) / 2; + + var config = ReactorConfig{ + .enabled = DEFAULT_REACTOR_ENABLED, + .continuation_pool = .{ + .size = default_continuation_workers, + .queue_capacity = DEFAULT_CONTINUATION_QUEUE_CAPACITY, + }, + .effector_pool = .{ + .size = default_effector_workers, + .queue_capacity = DEFAULT_EFFECTOR_QUEUE_CAPACITY, + }, + .compute_pool = .{ + .kind = .disabled, + .size = 0, + .queue_capacity = 0, + }, + }; + + if (raw) |reactor_raw| { + if (reactor_raw.enabled) |flag| { + config.enabled = flag; + } + + if (reactor_raw.continuation_pool) |pool_raw| { + if (pool_raw.size) |size| { + if (size == 0) return error.InvalidContinuationPoolSize; + config.continuation_pool.size = size; + } + if (pool_raw.queue_capacity) |capacity| { + if (capacity == 0) return error.InvalidContinuationQueueCapacity; + config.continuation_pool.queue_capacity = capacity; + } + } + + if (reactor_raw.effector_pool) |pool_raw| { + if (pool_raw.size) |size| { + if (size == 0) return error.InvalidEffectorPoolSize; + config.effector_pool.size = size; + } + if (pool_raw.queue_capacity) |capacity| { + if (capacity == 0) return error.InvalidEffectorQueueCapacity; + config.effector_pool.queue_capacity = capacity; + } + } + + if (reactor_raw.compute_pool) |pool_raw| { + if (pool_raw.type) |type_str| { + config.compute_pool.kind = try parseComputePoolKind(type_str); + } + + if (config.compute_pool.kind == .disabled) { + if (pool_raw.size) |size| { + if (size != 0) { + config.compute_pool.kind = .dedicated; + config.compute_pool.size = size; + } + } + } + + switch (config.compute_pool.kind) { + .disabled => { + config.compute_pool.size = 0; + config.compute_pool.queue_capacity = 0; + }, + .shared => { + if (pool_raw.size) |size| { + if (size == 0) return error.InvalidComputePoolSize; + config.compute_pool.size = size; + } else { + config.compute_pool.size = config.continuation_pool.size; + } + + config.compute_pool.queue_capacity = if (pool_raw.queue_capacity) |capacity| blk: { + if (capacity == 0) return error.InvalidComputeQueueCapacity; + break :blk capacity; + } else DEFAULT_COMPUTE_QUEUE_CAPACITY; + }, + .dedicated => { + const workers = if (pool_raw.size) |size| blk: { + if (size == 0) return error.InvalidComputePoolSize; + break :blk size; + } else default_compute_workers; + config.compute_pool.size = workers; + + config.compute_pool.queue_capacity = if (pool_raw.queue_capacity) |capacity| blk: { + if (capacity == 0) return error.InvalidComputeQueueCapacity; + break :blk capacity; + } else DEFAULT_COMPUTE_QUEUE_CAPACITY; + }, + } + } + } + + if (config.compute_pool.kind == .disabled) { + config.compute_pool.size = 0; + config.compute_pool.queue_capacity = 0; + } else { + if (config.compute_pool.queue_capacity == 0) { + config.compute_pool.queue_capacity = DEFAULT_COMPUTE_QUEUE_CAPACITY; + } + if (config.compute_pool.size == 0) { + config.compute_pool.size = default_compute_workers; + } + } + + return config; +} + +fn parseComputePoolKind(value: []const u8) !ComputePoolKind { + if (std.mem.eql(u8, value, "disabled")) return .disabled; + if (std.mem.eql(u8, value, "shared")) return .shared; + if (std.mem.eql(u8, value, "dedicated")) return .dedicated; + return error.InvalidComputePoolType; +} + fn buildObservabilityConfig( allocator: std.mem.Allocator, raw: ?RawObservabilityConfig, diff --git a/src/zerver/runtime/global.zig b/src/zerver/runtime/global.zig index abdf666..c8d374e 100644 --- a/src/zerver/runtime/global.zig +++ b/src/zerver/runtime/global.zig @@ -6,6 +6,10 @@ pub fn set(resources: *resources_mod.RuntimeResources) void { global_resources = resources; } +pub fn maybeGet() ?*resources_mod.RuntimeResources { + return global_resources; +} + pub fn get() *resources_mod.RuntimeResources { return global_resources orelse @panic("runtime resources not initialized"); } diff --git a/src/zerver/runtime/listener.zig b/src/zerver/runtime/listener.zig index 0576564..9eaa561 100644 --- a/src/zerver/runtime/listener.zig +++ b/src/zerver/runtime/listener.zig @@ -107,7 +107,7 @@ fn handleConnection( }; slog.info("handleRequest completed", &.{ - slog.Attr.string("result", @tagName(response_result)), + slog.Attr.enumeration("result", response_result), }); // Send response based on type diff --git a/src/zerver/runtime/reactor/effectors.zig b/src/zerver/runtime/reactor/effectors.zig new file mode 100644 index 0000000..0828c9b --- /dev/null +++ b/src/zerver/runtime/reactor/effectors.zig @@ -0,0 +1,260 @@ +const std = @import("std"); +const types = @import("../../core/types.zig"); +const libuv = @import("libuv.zig"); +const job = @import("job_system.zig"); +const task_system = @import("task_system.zig"); + +pub const DispatchError = error{ + UnsupportedEffect, +}; + +pub const Context = struct { + loop: *libuv.Loop, + jobs: *job.JobSystem, + compute_jobs: ?*job.JobSystem = null, + accelerator_jobs: ?*job.JobSystem = null, + kv_cache: ?*anyopaque = null, + task_system: ?*task_system.TaskSystem = null, +}; + +pub const HttpGetHandler = *const fn (*Context, types.HttpGet) DispatchError!types.EffectResult; +pub const HttpHeadHandler = *const fn (*Context, types.HttpHead) DispatchError!types.EffectResult; +pub const HttpPostHandler = *const fn (*Context, types.HttpPost) DispatchError!types.EffectResult; +pub const HttpPutHandler = *const fn (*Context, types.HttpPut) DispatchError!types.EffectResult; +pub const HttpDeleteHandler = *const fn (*Context, types.HttpDelete) DispatchError!types.EffectResult; +pub const HttpOptionsHandler = *const fn (*Context, types.HttpOptions) DispatchError!types.EffectResult; +pub const HttpTraceHandler = *const fn (*Context, types.HttpTrace) DispatchError!types.EffectResult; +pub const HttpConnectHandler = *const fn (*Context, types.HttpConnect) DispatchError!types.EffectResult; +pub const HttpPatchHandler = *const fn (*Context, types.HttpPatch) DispatchError!types.EffectResult; +pub const DbGetHandler = *const fn (*Context, types.DbGet) DispatchError!types.EffectResult; +pub const DbPutHandler = *const fn (*Context, types.DbPut) DispatchError!types.EffectResult; +pub const DbDelHandler = *const fn (*Context, types.DbDel) DispatchError!types.EffectResult; +pub const DbScanHandler = *const fn (*Context, types.DbScan) DispatchError!types.EffectResult; +pub const FileJsonReadHandler = *const fn (*Context, types.FileJsonRead) DispatchError!types.EffectResult; +pub const FileJsonWriteHandler = *const fn (*Context, types.FileJsonWrite) DispatchError!types.EffectResult; +pub const ComputeTaskHandler = *const fn (*Context, types.ComputeTask) DispatchError!types.EffectResult; +pub const AcceleratorTaskHandler = *const fn (*Context, types.AcceleratorTask) DispatchError!types.EffectResult; +pub const KvCacheGetHandler = *const fn (*Context, types.KvCacheGet) DispatchError!types.EffectResult; +pub const KvCacheSetHandler = *const fn (*Context, types.KvCacheSet) DispatchError!types.EffectResult; +pub const KvCacheDeleteHandler = *const fn (*Context, types.KvCacheDelete) DispatchError!types.EffectResult; + +pub const EffectHandlers = struct { + http_get: HttpGetHandler = defaultHttpGetHandler, + http_head: HttpHeadHandler = defaultHttpHeadHandler, + http_post: HttpPostHandler = defaultHttpPostHandler, + http_put: HttpPutHandler = defaultHttpPutHandler, + http_delete: HttpDeleteHandler = defaultHttpDeleteHandler, + http_options: HttpOptionsHandler = defaultHttpOptionsHandler, + http_trace: HttpTraceHandler = defaultHttpTraceHandler, + http_connect: HttpConnectHandler = defaultHttpConnectHandler, + http_patch: HttpPatchHandler = defaultHttpPatchHandler, + db_get: DbGetHandler = defaultDbGetHandler, + db_put: DbPutHandler = defaultDbPutHandler, + db_del: DbDelHandler = defaultDbDelHandler, + db_scan: DbScanHandler = defaultDbScanHandler, + file_json_read: FileJsonReadHandler = defaultFileJsonReadHandler, + file_json_write: FileJsonWriteHandler = defaultFileJsonWriteHandler, + compute_task: ComputeTaskHandler = defaultComputeTaskHandler, + accelerator_task: AcceleratorTaskHandler = defaultAcceleratorTaskHandler, + kv_cache_get: KvCacheGetHandler = defaultKvCacheGetHandler, + kv_cache_set: KvCacheSetHandler = defaultKvCacheSetHandler, + kv_cache_delete: KvCacheDeleteHandler = defaultKvCacheDeleteHandler, +}; + +pub const EffectDispatcher = struct { + handlers: EffectHandlers, + + pub fn init() EffectDispatcher { + return .{ .handlers = .{} }; + } + + pub fn setHttpGetHandler(self: *EffectDispatcher, handler: HttpGetHandler) void { + self.handlers.http_get = handler; + } + + pub fn setHttpHeadHandler(self: *EffectDispatcher, handler: HttpHeadHandler) void { + self.handlers.http_head = handler; + } + + pub fn setHttpPostHandler(self: *EffectDispatcher, handler: HttpPostHandler) void { + self.handlers.http_post = handler; + } + + pub fn setHttpPutHandler(self: *EffectDispatcher, handler: HttpPutHandler) void { + self.handlers.http_put = handler; + } + + pub fn setHttpDeleteHandler(self: *EffectDispatcher, handler: HttpDeleteHandler) void { + self.handlers.http_delete = handler; + } + + pub fn setHttpOptionsHandler(self: *EffectDispatcher, handler: HttpOptionsHandler) void { + self.handlers.http_options = handler; + } + + pub fn setHttpTraceHandler(self: *EffectDispatcher, handler: HttpTraceHandler) void { + self.handlers.http_trace = handler; + } + + pub fn setHttpConnectHandler(self: *EffectDispatcher, handler: HttpConnectHandler) void { + self.handlers.http_connect = handler; + } + + pub fn setHttpPatchHandler(self: *EffectDispatcher, handler: HttpPatchHandler) void { + self.handlers.http_patch = handler; + } + + pub fn setDbGetHandler(self: *EffectDispatcher, handler: DbGetHandler) void { + self.handlers.db_get = handler; + } + + pub fn setDbPutHandler(self: *EffectDispatcher, handler: DbPutHandler) void { + self.handlers.db_put = handler; + } + + pub fn setDbDelHandler(self: *EffectDispatcher, handler: DbDelHandler) void { + self.handlers.db_del = handler; + } + + pub fn setDbScanHandler(self: *EffectDispatcher, handler: DbScanHandler) void { + self.handlers.db_scan = handler; + } + + pub fn setFileJsonReadHandler(self: *EffectDispatcher, handler: FileJsonReadHandler) void { + self.handlers.file_json_read = handler; + } + + pub fn setFileJsonWriteHandler(self: *EffectDispatcher, handler: FileJsonWriteHandler) void { + self.handlers.file_json_write = handler; + } + + pub fn setComputeTaskHandler(self: *EffectDispatcher, handler: ComputeTaskHandler) void { + self.handlers.compute_task = handler; + } + + pub fn setAcceleratorTaskHandler(self: *EffectDispatcher, handler: AcceleratorTaskHandler) void { + self.handlers.accelerator_task = handler; + } + + pub fn setKvCacheGetHandler(self: *EffectDispatcher, handler: KvCacheGetHandler) void { + self.handlers.kv_cache_get = handler; + } + + pub fn setKvCacheSetHandler(self: *EffectDispatcher, handler: KvCacheSetHandler) void { + self.handlers.kv_cache_set = handler; + } + + pub fn setKvCacheDeleteHandler(self: *EffectDispatcher, handler: KvCacheDeleteHandler) void { + self.handlers.kv_cache_delete = handler; + } + + pub fn dispatch(self: *EffectDispatcher, ctx: *Context, effect: types.Effect) DispatchError!types.EffectResult { + return switch (effect) { + .http_get => |payload| try self.handlers.http_get(ctx, payload), + .http_head => |payload| try self.handlers.http_head(ctx, payload), + .http_post => |payload| try self.handlers.http_post(ctx, payload), + .http_put => |payload| try self.handlers.http_put(ctx, payload), + .http_delete => |payload| try self.handlers.http_delete(ctx, payload), + .http_options => |payload| try self.handlers.http_options(ctx, payload), + .http_trace => |payload| try self.handlers.http_trace(ctx, payload), + .http_connect => |payload| try self.handlers.http_connect(ctx, payload), + .http_patch => |payload| try self.handlers.http_patch(ctx, payload), + .db_get => |payload| try self.handlers.db_get(ctx, payload), + .db_put => |payload| try self.handlers.db_put(ctx, payload), + .db_del => |payload| try self.handlers.db_del(ctx, payload), + .db_scan => |payload| try self.handlers.db_scan(ctx, payload), + .file_json_read => |payload| try self.handlers.file_json_read(ctx, payload), + .file_json_write => |payload| try self.handlers.file_json_write(ctx, payload), + .compute_task => |payload| try self.handlers.compute_task(ctx, payload), + .accelerator_task => |payload| try self.handlers.accelerator_task(ctx, payload), + .kv_cache_get => |payload| try self.handlers.kv_cache_get(ctx, payload), + .kv_cache_set => |payload| try self.handlers.kv_cache_set(ctx, payload), + .kv_cache_delete => |payload| try self.handlers.kv_cache_delete(ctx, payload), + }; + } +}; + +fn unsupported(comptime label: []const u8) DispatchError { + _ = label; + return DispatchError.UnsupportedEffect; +} + +fn defaultHttpGetHandler(_: *Context, _: types.HttpGet) DispatchError!types.EffectResult { + return unsupported("http_get"); +} + +fn defaultHttpHeadHandler(_: *Context, _: types.HttpHead) DispatchError!types.EffectResult { + return unsupported("http_head"); +} + +fn defaultHttpPostHandler(_: *Context, _: types.HttpPost) DispatchError!types.EffectResult { + return unsupported("http_post"); +} + +fn defaultHttpPutHandler(_: *Context, _: types.HttpPut) DispatchError!types.EffectResult { + return unsupported("http_put"); +} + +fn defaultHttpDeleteHandler(_: *Context, _: types.HttpDelete) DispatchError!types.EffectResult { + return unsupported("http_delete"); +} + +fn defaultHttpOptionsHandler(_: *Context, _: types.HttpOptions) DispatchError!types.EffectResult { + return unsupported("http_options"); +} + +fn defaultHttpTraceHandler(_: *Context, _: types.HttpTrace) DispatchError!types.EffectResult { + return unsupported("http_trace"); +} + +fn defaultHttpConnectHandler(_: *Context, _: types.HttpConnect) DispatchError!types.EffectResult { + return unsupported("http_connect"); +} + +fn defaultHttpPatchHandler(_: *Context, _: types.HttpPatch) DispatchError!types.EffectResult { + return unsupported("http_patch"); +} + +fn defaultDbGetHandler(_: *Context, _: types.DbGet) DispatchError!types.EffectResult { + return unsupported("db_get"); +} + +fn defaultDbPutHandler(_: *Context, _: types.DbPut) DispatchError!types.EffectResult { + return unsupported("db_put"); +} + +fn defaultDbDelHandler(_: *Context, _: types.DbDel) DispatchError!types.EffectResult { + return unsupported("db_del"); +} + +fn defaultDbScanHandler(_: *Context, _: types.DbScan) DispatchError!types.EffectResult { + return unsupported("db_scan"); +} + +fn defaultFileJsonReadHandler(_: *Context, _: types.FileJsonRead) DispatchError!types.EffectResult { + return unsupported("file_json_read"); +} + +fn defaultFileJsonWriteHandler(_: *Context, _: types.FileJsonWrite) DispatchError!types.EffectResult { + return unsupported("file_json_write"); +} + +fn defaultComputeTaskHandler(_: *Context, _: types.ComputeTask) DispatchError!types.EffectResult { + return unsupported("compute_task"); +} + +fn defaultAcceleratorTaskHandler(_: *Context, _: types.AcceleratorTask) DispatchError!types.EffectResult { + return unsupported("accelerator_task"); +} + +fn defaultKvCacheGetHandler(_: *Context, _: types.KvCacheGet) DispatchError!types.EffectResult { + return unsupported("kv_cache_get"); +} + +fn defaultKvCacheSetHandler(_: *Context, _: types.KvCacheSet) DispatchError!types.EffectResult { + return unsupported("kv_cache_set"); +} + +fn defaultKvCacheDeleteHandler(_: *Context, _: types.KvCacheDelete) DispatchError!types.EffectResult { + return unsupported("kv_cache_delete"); +} diff --git a/src/zerver/runtime/reactor/job_system.zig b/src/zerver/runtime/reactor/job_system.zig new file mode 100644 index 0000000..83ca978 --- /dev/null +++ b/src/zerver/runtime/reactor/job_system.zig @@ -0,0 +1,319 @@ +const std = @import("std"); +const slog = @import("../../observability/slog.zig"); + +const AtomicOrder = std.builtin.AtomicOrder; + +pub const SubmitError = error{ + ShuttingDown, + OutOfMemory, + QueueFull, +}; + +pub const JobFn = *const fn (*anyopaque) void; + +pub const Job = struct { + callback: JobFn, + ctx: *anyopaque, +}; + +pub const InitOptions = struct { + allocator: std.mem.Allocator, + worker_count: usize, + queue_capacity: usize = 0, + label: []const u8 = "job_system", +}; + +pub const JobSystem = struct { + allocator: std.mem.Allocator, + mutex: std.Thread.Mutex = .{}, + cond: std.Thread.Condition = .{}, + queue: JobQueue, + accepting: std.atomic.Value(bool) = std.atomic.Value(bool).init(true), + workers: []std.Thread, + queue_label: []const u8, + + pub fn init(self: *JobSystem, options: InitOptions) !void { + self.* = .{ + .allocator = options.allocator, + .mutex = .{}, + .cond = .{}, + .queue = JobQueue.init(options.allocator, options.queue_capacity), + .accepting = std.atomic.Value(bool).init(true), + .workers = &[_]std.Thread{}, + .queue_label = options.label, + }; + + slog.debug("job_system_init", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("workers", @as(u64, @intCast(options.worker_count))), + slog.Attr.uint("capacity", @as(u64, @intCast(options.queue_capacity))), + }); + + errdefer self.queue.deinit(); + + if (options.queue_capacity > 0) try self.queue.ensureCapacity(options.queue_capacity); + + if (options.worker_count == 0) { + slog.debug("job_system_no_workers", &.{ + slog.Attr.string("queue", self.queue_label), + }); + self.workers = &[_]std.Thread{}; + return; + } + + self.workers = try options.allocator.alloc(std.Thread, options.worker_count); + errdefer options.allocator.free(self.workers); + + var index: usize = 0; + while (index < options.worker_count) : (index += 1) { + slog.debug("job_worker_spawn", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(index))), + }); + self.workers[index] = try std.Thread.spawn(.{}, workerMain, .{ self, index }); + } + } + + pub fn deinit(self: *JobSystem) void { + slog.debug("job_system_deinit", &.{ + slog.Attr.string("queue", self.queue_label), + }); + self.shutdown(); + for (self.workers) |*worker| { + worker.join(); + } + if (self.workers.len > 0) self.allocator.free(self.workers); + self.queue.deinit(); + } + + pub fn shutdown(self: *JobSystem) void { + const previous = self.accepting.swap(false, AtomicOrder.seq_cst); + if (!previous) return; + + slog.debug("job_system_shutdown", &.{ + slog.Attr.string("queue", self.queue_label), + }); + + self.mutex.lock(); + defer self.mutex.unlock(); + self.cond.broadcast(); + } + + pub fn submit(self: *JobSystem, job: Job) SubmitError!void { + if (!self.accepting.load(AtomicOrder.seq_cst)) { + slog.debug("job_enqueue_rejected", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job.ctx)))), + }); + return SubmitError.ShuttingDown; + } + + self.mutex.lock(); + defer self.mutex.unlock(); + + const before_len = self.queue.count; + self.queue.write(job) catch |err| { + slog.err("job_enqueue_failed", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.string("error", @errorName(err)), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job.ctx)))), + slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(job.callback)))), + }); + return switch (err) { + error.QueueFull => SubmitError.QueueFull, + error.OutOfMemory => SubmitError.OutOfMemory, + }; + }; + const after_len = self.queue.count; + slog.debug("job_enqueued", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job.ctx)))), + slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(job.callback)))), + slog.Attr.uint("queued", @as(u64, @intCast(after_len))), + slog.Attr.uint("queued_prev", @as(u64, @intCast(before_len))), + }); + self.cond.signal(); + } + + fn workerMain(self: *JobSystem, worker_index: usize) !void { + slog.debug("job_worker_start", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + }); + defer slog.debug("job_worker_exit", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + }); + + while (true) { + const job_opt = self.nextJob(worker_index); + if (job_opt) |job| { + slog.debug("job_worker_execute", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job.ctx)))), + slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(job.callback)))), + }); + job.callback(job.ctx); + slog.debug("job_worker_complete", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job.ctx)))), + slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(job.callback)))), + }); + } else { + break; + } + } + } + + fn nextJob(self: *JobSystem, worker_index: usize) ?Job { + self.mutex.lock(); + defer self.mutex.unlock(); + + while (true) { + const job = self.queue.read() catch |err| { + if (err == error.Empty) { + if (!self.accepting.load(AtomicOrder.seq_cst)) { + slog.debug("job_worker_drain_complete", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + }); + return null; + } + slog.debug("job_worker_park", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + }); + self.cond.wait(&self.mutex); + slog.debug("job_worker_unpark", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + slog.Attr.uint("queued", @as(u64, @intCast(self.queue.count))), + }); + continue; + } + slog.err("job_queue_read_failed", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.string("error", @errorName(err)), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + }); + return null; + }; + slog.debug("job_worker_dequeue", &.{ + slog.Attr.string("queue", self.queue_label), + slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job.ctx)))), + slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(job.callback)))), + slog.Attr.uint("queued", @as(u64, @intCast(self.queue.count))), + }); + return job; + } + } + + pub fn label(self: *JobSystem) []const u8 { + return self.queue_label; + } +}; + +const JobQueue = struct { + allocator: std.mem.Allocator, + buffer: []Job, + head: usize, + tail: usize, + count: usize, + max_capacity: usize, + + fn init(allocator: std.mem.Allocator, max_capacity: usize) JobQueue { + return .{ + .allocator = allocator, + .buffer = &[_]Job{}, + .head = 0, + .tail = 0, + .count = 0, + .max_capacity = max_capacity, + }; + } + + fn deinit(self: *JobQueue) void { + if (self.buffer.len != 0) { + self.allocator.free(self.buffer); + self.buffer = &[_]Job{}; + } + self.head = 0; + self.tail = 0; + self.count = 0; + self.max_capacity = 0; + } + + fn write(self: *JobQueue, job: Job) error{ QueueFull, OutOfMemory }!void { + if (self.max_capacity != 0 and self.count == self.max_capacity) { + return error.QueueFull; + } + + if (self.buffer.len == 0) { + const initial = if (self.max_capacity != 0) + @min(self.max_capacity, @as(usize, 4)) + else + 4; + if (initial == 0) return error.QueueFull; + try self.ensureCapacity(initial); + } else if (self.count == self.buffer.len) { + const desired = if (self.buffer.len == 0) 4 else self.buffer.len * 2; + try self.ensureCapacity(desired); + } + + self.buffer[self.tail] = job; + self.tail = (self.tail + 1) % self.buffer.len; + self.count += 1; + } + + fn read(self: *JobQueue) error{Empty}!Job { + if (self.count == 0) return error.Empty; + + const job = self.buffer[self.head]; + self.head = (self.head + 1) % self.buffer.len; + self.count -= 1; + + if (self.count == 0) { + self.head = 0; + self.tail = 0; + } + + return job; + } + + fn ensureCapacity(self: *JobQueue, requested: usize) error{ OutOfMemory, QueueFull }!void { + const old_capacity = self.buffer.len; + var target = if (requested > old_capacity) requested else old_capacity; + + if (self.max_capacity != 0) { + if (target > self.max_capacity) { + target = self.max_capacity; + } + if (target == 0) return error.QueueFull; + if (target <= old_capacity) return error.QueueFull; + } else { + if (target < 4) target = 4; + if (target <= old_capacity) return; + } + + var new_buffer = try self.allocator.alloc(Job, target); + + if (self.count > 0 and old_capacity != 0) { + var index: usize = 0; + while (index < self.count) : (index += 1) { + const src_index = (self.head + index) % old_capacity; + new_buffer[index] = self.buffer[src_index]; + } + } + + if (old_capacity != 0) { + self.allocator.free(self.buffer); + } + + self.buffer = new_buffer; + self.head = 0; + self.tail = self.count; + } +}; diff --git a/src/zerver/runtime/reactor/join.zig b/src/zerver/runtime/reactor/join.zig new file mode 100644 index 0000000..0420470 --- /dev/null +++ b/src/zerver/runtime/reactor/join.zig @@ -0,0 +1,114 @@ +const std = @import("std"); +const types = @import("../../core/types.zig"); + +pub const Mode = types.Mode; +pub const Join = types.Join; + +pub const Status = enum { + success, + failure, +}; + +pub const Resolution = union(enum) { + Pending, + Resume: struct { + status: Status, + }, +}; + +pub const JoinConfig = struct { + mode: Mode, + join: Join, +}; + +pub const Completion = struct { + required: bool, + success: bool, +}; + +pub const JoinState = struct { + config: JoinConfig, + outstanding: usize, + required_remaining: usize, + success_seen: bool, + required_failure: bool, + resumed: bool, + + pub fn init(config: JoinConfig, total_effects: usize, required_effects: usize) JoinState { + std.debug.assert(total_effects > 0); + std.debug.assert(required_effects <= total_effects); + return .{ + .config = config, + .outstanding = total_effects, + .required_remaining = required_effects, + .success_seen = false, + .required_failure = false, + .resumed = false, + }; + } + + pub fn record(self: *JoinState, completion: Completion) Resolution { + if (self.resumed) return .Pending; + + std.debug.assert(self.outstanding > 0); + self.outstanding -= 1; + + if (completion.required and self.required_remaining > 0) { + self.required_remaining -= 1; + } + + if (completion.success) { + self.success_seen = true; + } else if (completion.required) { + self.required_failure = true; + } + + switch (self.config.join) { + .any => { + self.resumed = true; + const status: Status = if (completion.success) .success else .failure; + return .{ .Resume = .{ .status = status } }; + }, + .first_success => { + if (completion.success) { + self.resumed = true; + return .{ .Resume = .{ .status = .success } }; + } + if (self.required_failure) { + self.resumed = true; + return .{ .Resume = .{ .status = .failure } }; + } + if (self.outstanding == 0 and !self.success_seen) { + self.resumed = true; + return .{ .Resume = .{ .status = .failure } }; + } + }, + .all => { + if (self.required_failure) { + self.resumed = true; + return .{ .Resume = .{ .status = .failure } }; + } + if (self.outstanding == 0) { + self.resumed = true; + return .{ .Resume = .{ .status = .success } }; + } + }, + .all_required => { + if (self.required_failure) { + self.resumed = true; + return .{ .Resume = .{ .status = .failure } }; + } + if (self.required_remaining == 0) { + self.resumed = true; + return .{ .Resume = .{ .status = .success } }; + } + }, + } + + return .Pending; + } + + pub fn isResumed(self: *const JoinState) bool { + return self.resumed; + } +}; diff --git a/src/zerver/runtime/reactor/libuv.zig b/src/zerver/runtime/reactor/libuv.zig new file mode 100644 index 0000000..e93d442 --- /dev/null +++ b/src/zerver/runtime/reactor/libuv.zig @@ -0,0 +1,53 @@ +const std = @import("std"); + +const c = @cImport({ + @cInclude("uv.h"); +}); + +pub const Error = error{ + LoopInitFailed, + LoopCloseFailed, +}; + +pub const RunMode = enum { + default, + once, + nowait, +}; + +pub const Loop = struct { + inner: c.uv_loop_t, + + pub fn init() Error!Loop { + var instance = Loop{ .inner = undefined }; + if (c.uv_loop_init(&instance.inner) != 0) { + return Error.LoopInitFailed; + } + return instance; + } + + pub fn deinit(self: *Loop) Error!void { + const rc = c.uv_loop_close(&self.inner); + if (rc != 0) { + return Error.LoopCloseFailed; + } + } + + pub fn run(self: *Loop, mode: RunMode) bool { + const uv_mode = switch (mode) { + .default => c.UV_RUN_DEFAULT, + .once => c.UV_RUN_ONCE, + .nowait => c.UV_RUN_NOWAIT, + }; + const result = c.uv_run(&self.inner, uv_mode); + return result != 0; + } + + pub fn stop(self: *Loop) void { + c.uv_stop(&self.inner); + } + + pub fn ptr(self: *Loop) *c.uv_loop_t { + return &self.inner; + } +}; diff --git a/src/zerver/runtime/reactor/saga.zig b/src/zerver/runtime/reactor/saga.zig new file mode 100644 index 0000000..59c4b50 --- /dev/null +++ b/src/zerver/runtime/reactor/saga.zig @@ -0,0 +1,30 @@ +const std = @import("std"); +const types = @import("../../core/types.zig"); + +/// Saga support is deferred to a later phase; this stub exists so higher layers +/// can start threading compensation metadata without hard dependencies. +pub const SagaError = error{Unimplemented}; + +pub const SagaLog = struct { + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) SagaLog { + return .{ .allocator = allocator }; + } + + pub fn deinit(_: *SagaLog) void {} + + pub fn record(_: *SagaLog, _: types.Compensation) SagaError!void { + return SagaError.Unimplemented; + } + + pub fn pop(_: *SagaLog) ?types.Compensation { + return null; + } + + pub fn len(_: *SagaLog) usize { + return 0; + } + + pub fn clear(_: *SagaLog) void {} +}; diff --git a/src/zerver/runtime/reactor/task_system.zig b/src/zerver/runtime/reactor/task_system.zig new file mode 100644 index 0000000..d5f06b3 --- /dev/null +++ b/src/zerver/runtime/reactor/task_system.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const job = @import("job_system.zig"); +const slog = @import("../../observability/slog.zig"); + +pub const TaskSystemError = job.SubmitError || error{NoComputePool}; + +pub const ComputePoolKind = enum { + disabled, + shared, + dedicated, +}; + +pub const TaskSystemConfig = struct { + allocator: std.mem.Allocator, + continuation_workers: usize, + continuation_queue_capacity: usize = 0, + compute_kind: ComputePoolKind = .disabled, + compute_workers: usize = 0, + compute_queue_capacity: usize = 0, +}; + +pub const TaskSystem = struct { + continuation: job.JobSystem = undefined, + compute: job.JobSystem = undefined, + has_compute: bool = false, + compute_kind: ComputePoolKind = .disabled, + + pub fn init(self: *TaskSystem, config: TaskSystemConfig) !void { + self.compute_kind = config.compute_kind; + self.has_compute = false; + + try self.continuation.init(.{ + .allocator = config.allocator, + .worker_count = config.continuation_workers, + .queue_capacity = config.continuation_queue_capacity, + .label = "continuation_jobs", + }); + errdefer self.continuation.deinit(); + + switch (config.compute_kind) { + .disabled => {}, + .shared => {}, + .dedicated => { + if (config.compute_workers == 0) { + self.compute_kind = .disabled; + } else { + try self.compute.init(.{ + .allocator = config.allocator, + .worker_count = config.compute_workers, + .queue_capacity = config.compute_queue_capacity, + .label = "compute_jobs", + }); + errdefer self.compute.deinit(); + self.has_compute = true; + } + }, + } + + slog.debug("task_system_init", &.{ + slog.Attr.string("continuation_queue", self.continuation.label()), + slog.Attr.string("compute_kind", @tagName(self.compute_kind)), + slog.Attr.bool("has_compute", self.has_compute), + }); + } + + pub fn deinit(self: *TaskSystem) void { + if (self.compute_kind == .dedicated and self.has_compute) { + self.compute.deinit(); + } + self.continuation.deinit(); + } + + pub fn shutdown(self: *TaskSystem) void { + if (self.compute_kind == .dedicated and self.has_compute) { + self.compute.shutdown(); + } + self.continuation.shutdown(); + } + + pub fn submitContinuation(self: *TaskSystem, task: job.Job) TaskSystemError!void { + slog.debug("task_submit_continuation", &.{ + slog.Attr.string("queue", self.continuation.label()), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(task.ctx)))), + slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(task.callback)))), + }); + self.continuation.submit(task) catch |err| { + slog.err("task_submit_continuation_failed", &.{ + slog.Attr.string("queue", self.continuation.label()), + slog.Attr.string("error", @errorName(err)), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(task.ctx)))), + }); + return err; + }; + } + + pub fn submitCompute(self: *TaskSystem, task: job.Job) TaskSystemError!void { + slog.debug("task_submit_compute", &.{ + slog.Attr.string("mode", @tagName(self.compute_kind)), + slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(task.ctx)))), + slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(task.callback)))), + }); + return switch (self.compute_kind) { + .disabled => error.NoComputePool, + .shared => self.continuation.submit(task) catch |err| { + slog.err("task_submit_compute_failed", &.{ + slog.Attr.string("queue", self.continuation.label()), + slog.Attr.string("error", @errorName(err)), + slog.Attr.string("mode", "shared"), + }); + return err; + }, + .dedicated => blk: { + if (!self.has_compute) break :blk error.NoComputePool; + self.compute.submit(task) catch |err| { + slog.err("task_submit_compute_failed", &.{ + slog.Attr.string("queue", self.compute.label()), + slog.Attr.string("error", @errorName(err)), + slog.Attr.string("mode", "dedicated"), + }); + return err; + }; + slog.debug("task_submit_compute_queued", &.{ + slog.Attr.string("queue", self.compute.label()), + }); + break :blk {}; + }, + }; + } + + pub fn continuationJobs(self: *TaskSystem) *job.JobSystem { + return &self.continuation; + } + + pub fn computeJobs(self: *TaskSystem) ?*job.JobSystem { + return switch (self.compute_kind) { + .disabled => null, + .shared => &self.continuation, + .dedicated => if (self.has_compute) &self.compute else null, + }; + } + + pub fn hasComputePool(self: *TaskSystem) bool { + return self.compute_kind != .disabled; + } +}; diff --git a/src/zerver/runtime/resources.zig b/src/zerver/runtime/resources.zig index dbc8b07..e21c180 100644 --- a/src/zerver/runtime/resources.zig +++ b/src/zerver/runtime/resources.zig @@ -1,9 +1,129 @@ const std = @import("std"); const config_mod = @import("config.zig"); const sql = @import("../sql/mod.zig"); +const task_system = @import("reactor/task_system.zig"); +const job_system = @import("reactor/job_system.zig"); +const effectors = @import("reactor/effectors.zig"); +const libuv = @import("reactor/libuv.zig"); const sqlite_driver = &sql.dialects.sqlite.driver.driver; +const ReactorResources = struct { + enabled: bool = false, + task_system: task_system.TaskSystem = undefined, + effector_jobs: job_system.JobSystem = undefined, + has_task_system: bool = false, + has_effector_jobs: bool = false, + dispatcher: effectors.EffectDispatcher = effectors.EffectDispatcher.init(), + loop: libuv.Loop = undefined, + loop_initialized: bool = false, + + fn init(self: *ReactorResources, allocator: std.mem.Allocator, cfg: config_mod.ReactorConfig) !void { + self.* = .{ + .enabled = cfg.enabled, + .has_task_system = false, + .has_effector_jobs = false, + .task_system = undefined, + .effector_jobs = undefined, + .dispatcher = effectors.EffectDispatcher.init(), + .loop = undefined, + .loop_initialized = false, + }; + + if (!cfg.enabled) return; + + errdefer self.deinit(); + + self.loop = try libuv.Loop.init(); + self.loop_initialized = true; + + try self.effector_jobs.init(.{ + .allocator = allocator, + .worker_count = cfg.effector_pool.size, + .queue_capacity = cfg.effector_pool.queue_capacity, + .label = "effector_jobs", + }); + self.has_effector_jobs = true; + + try self.task_system.init(.{ + .allocator = allocator, + .continuation_workers = cfg.continuation_pool.size, + .continuation_queue_capacity = cfg.continuation_pool.queue_capacity, + .compute_kind = convertComputeKind(cfg.compute_pool.kind), + .compute_workers = cfg.compute_pool.size, + .compute_queue_capacity = cfg.compute_pool.queue_capacity, + }); + self.has_task_system = true; + } + + fn shutdown(self: *ReactorResources) void { + if (!self.enabled) return; + if (self.loop_initialized) self.loop.stop(); + if (self.has_task_system) self.task_system.shutdown(); + if (self.has_effector_jobs) self.effector_jobs.shutdown(); + } + + fn deinit(self: *ReactorResources) void { + if (!self.enabled) return; + self.shutdown(); + if (self.has_task_system) { + self.task_system.deinit(); + self.has_task_system = false; + } + if (self.has_effector_jobs) { + self.effector_jobs.deinit(); + self.has_effector_jobs = false; + } + if (self.loop_initialized) { + self.loop.deinit() catch {}; + self.loop_initialized = false; + } + self.dispatcher = effectors.EffectDispatcher.init(); + self.enabled = false; + } + + fn taskSystem(self: *ReactorResources) ?*task_system.TaskSystem { + if (!self.enabled or !self.has_task_system) return null; + return &self.task_system; + } + + fn effectorJobs(self: *ReactorResources) ?*job_system.JobSystem { + if (!self.enabled or !self.has_effector_jobs) return null; + return &self.effector_jobs; + } + + fn effectDispatcher(self: *ReactorResources) ?*effectors.EffectDispatcher { + if (!self.enabled) return null; + return &self.dispatcher; + } + + fn loopPtr(self: *ReactorResources) ?*libuv.Loop { + if (!self.enabled or !self.loop_initialized) return null; + return &self.loop; + } + + fn context(self: *ReactorResources) ?effectors.Context { + if (!self.enabled or !self.has_effector_jobs or !self.loop_initialized) return null; + const compute_jobs = if (self.has_task_system) self.task_system.computeJobs() else null; + return effectors.Context{ + .loop = &self.loop, + .jobs = &self.effector_jobs, + .compute_jobs = compute_jobs, + .accelerator_jobs = null, + .kv_cache = null, + .task_system = if (self.has_task_system) &self.task_system else null, + }; + } + + fn convertComputeKind(kind: config_mod.ComputePoolKind) task_system.ComputePoolKind { + return switch (kind) { + .disabled => .disabled, + .shared => .shared, + .dedicated => .dedicated, + }; + } +}; + pub const RuntimeResources = struct { allocator: std.mem.Allocator, config: config_mod.AppConfig, @@ -13,6 +133,7 @@ pub const RuntimeResources = struct { pool_cond: std.Thread.Condition = .{}, thread_pool: std.Thread.Pool, shutting_down: bool = false, + reactor: ReactorResources = .{}, pub fn init(self: *RuntimeResources, allocator: std.mem.Allocator, config: config_mod.AppConfig) !void { self.allocator = allocator; @@ -49,14 +170,19 @@ pub const RuntimeResources = struct { }); errdefer self.thread_pool.deinit(); + self.reactor = .{}; + errdefer self.reactor.deinit(); + try self.reactor.init(allocator, config.reactor); + self.pool_mutex = .{}; self.pool_cond = .{}; self.shutting_down = false; } pub fn deinit(self: *RuntimeResources) void { + self.reactor.shutdown(); + self.pool_mutex.lock(); - defer self.pool_mutex.unlock(); self.shutting_down = true; self.pool_cond.broadcast(); @@ -65,6 +191,9 @@ pub const RuntimeResources = struct { self.allocator.destroy(conn_ptr); } self.connections.deinit(self.allocator); + self.pool_mutex.unlock(); + + self.reactor.deinit(); self.thread_pool.deinit(); self.registry.deinit(); @@ -75,6 +204,30 @@ pub const RuntimeResources = struct { return &self.config; } + pub fn reactorEnabled(self: *RuntimeResources) bool { + return self.reactor.enabled; + } + + pub fn reactorTaskSystem(self: *RuntimeResources) ?*task_system.TaskSystem { + return self.reactor.taskSystem(); + } + + pub fn reactorEffectorJobs(self: *RuntimeResources) ?*job_system.JobSystem { + return self.reactor.effectorJobs(); + } + + pub fn reactorEffectDispatcher(self: *RuntimeResources) ?*effectors.EffectDispatcher { + return self.reactor.effectDispatcher(); + } + + pub fn reactorLoop(self: *RuntimeResources) ?*libuv.Loop { + return self.reactor.loopPtr(); + } + + pub fn reactorEffectContext(self: *RuntimeResources) ?effectors.Context { + return self.reactor.context(); + } + pub const ConnectionLease = struct { resources: ?*RuntimeResources, conn_ptr: *sql.db.Connection, diff --git a/tests/libuv_smoke.zig b/tests/libuv_smoke.zig new file mode 100644 index 0000000..0809293 --- /dev/null +++ b/tests/libuv_smoke.zig @@ -0,0 +1,195 @@ +const std = @import("std"); +const log = std.log.scoped(.libuv_smoke); + +const c = @cImport({ + @cInclude("uv.h"); +}); + +const LibuvError = error{ + LoopInitFailed, + LoopCloseFailed, + TimerInitFailed, + TimerStartFailed, + TimerDidNotFire, + AsyncInitFailed, + AsyncSendFailed, + AsyncDidNotFire, + WorkQueueFailed, + WorkDidNotRun, + WorkCompletionFailed, + RunFailed, +}; + +const Scenario = struct { + name: []const u8, + run: *const fn () LibuvError!void, +}; + +const scenarios = [_]Scenario{ + .{ .name = "timer fires and closes", .run = runTimerScenario }, + .{ .name = "async handle dispatch", .run = runAsyncScenario }, + .{ .name = "threadpool work queue", .run = runWorkScenario }, +}; + +const TimerState = struct { + fired: bool = false, +}; + +const AsyncState = struct { + triggered: bool = false, +}; + +const WorkState = struct { + executed: bool = false, + after_called: bool = false, + after_status: c_int = 0, +}; + +fn closeHandle(loop: *c.uv_loop_t, handle: anytype) void { + const base_handle: *c.uv_handle_t = @ptrCast(handle); + c.uv_close(base_handle, null); + _ = c.uv_run(loop, c.UV_RUN_DEFAULT); +} + +fn timerCallback(handle: ?*c.uv_timer_t) callconv(.c) void { + const timer = handle.?; + log.debug("timer callback fired", .{}); + if (timer.data) |raw_ptr| { + const state_ptr: *TimerState = @ptrCast(raw_ptr); + state_ptr.fired = true; + } + const base_handle: *c.uv_handle_t = @ptrCast(timer); + c.uv_close(base_handle, null); +} + +fn asyncCallback(handle: ?*c.uv_async_t) callconv(.c) void { + const async = handle.?; + log.debug("async callback executed", .{}); + if (async.data) |raw_ptr| { + const state_ptr: *AsyncState = @ptrCast(raw_ptr); + state_ptr.triggered = true; + } + const base_handle: *c.uv_handle_t = @ptrCast(async); + c.uv_close(base_handle, null); +} + +fn workExecute(req: ?*c.uv_work_t) callconv(.c) void { + const work = req.?; + log.debug("work execute on thread", .{}); + if (work.data) |payload| { + const aligned_payload: *align(@alignOf(WorkState)) anyopaque = @alignCast(payload); + const state_ptr: *WorkState = @ptrCast(aligned_payload); + state_ptr.executed = true; + } +} + +fn workAfter(req: ?*c.uv_work_t, status: c_int) callconv(.c) void { + const work = req.?; + log.debug("work completion status={d}", .{status}); + if (work.data) |payload| { + const aligned_payload: *align(@alignOf(WorkState)) anyopaque = @alignCast(payload); + const state_ptr: *WorkState = @ptrCast(aligned_payload); + state_ptr.after_called = true; + state_ptr.after_status = status; + } +} + +fn runTimerScenario() LibuvError!void { + log.info("Running timer scenario", .{}); + var loop: c.uv_loop_t = undefined; + if (c.uv_loop_init(&loop) != 0) return LibuvError.LoopInitFailed; + var loop_open = true; + errdefer { + if (loop_open) _ = c.uv_loop_close(&loop); + } + + var state = TimerState{}; + var timer: c.uv_timer_t = undefined; + if (c.uv_timer_init(&loop, &timer) != 0) return LibuvError.TimerInitFailed; + var timer_open = true; + errdefer if (timer_open) closeHandle(&loop, &timer); + + timer.data = @ptrCast(&state); + const start_status = c.uv_timer_start(&timer, timerCallback, 5, 0); + if (start_status != 0) return LibuvError.TimerStartFailed; + log.info("uv_timer_start scheduled timeout={d}ms", .{5}); + + const run_status = c.uv_run(&loop, c.UV_RUN_DEFAULT); + if (run_status != 0) return LibuvError.RunFailed; + timer_open = false; + + if (!state.fired) return LibuvError.TimerDidNotFire; + + const close_status = c.uv_loop_close(&loop); + loop_open = false; + if (close_status != 0) return LibuvError.LoopCloseFailed; + log.info("Timer scenario complete", .{}); +} + +fn runAsyncScenario() LibuvError!void { + log.info("Running async scenario", .{}); + var loop: c.uv_loop_t = undefined; + if (c.uv_loop_init(&loop) != 0) return LibuvError.LoopInitFailed; + var loop_open = true; + errdefer { + if (loop_open) _ = c.uv_loop_close(&loop); + } + + var state = AsyncState{}; + var async_handle: c.uv_async_t = undefined; + if (c.uv_async_init(&loop, &async_handle, asyncCallback) != 0) return LibuvError.AsyncInitFailed; + var async_open = true; + errdefer if (async_open) closeHandle(&loop, &async_handle); + + async_handle.data = @ptrCast(&state); + const send_status = c.uv_async_send(&async_handle); + if (send_status != 0) return LibuvError.AsyncSendFailed; + log.info("uv_async_send dispatched", .{}); + + const run_status = c.uv_run(&loop, c.UV_RUN_DEFAULT); + if (run_status != 0) return LibuvError.RunFailed; + async_open = false; + + if (!state.triggered) return LibuvError.AsyncDidNotFire; + + const close_status = c.uv_loop_close(&loop); + loop_open = false; + if (close_status != 0) return LibuvError.LoopCloseFailed; + log.info("Async scenario complete", .{}); +} + +fn runWorkScenario() LibuvError!void { + log.info("Running work queue scenario", .{}); + var loop: c.uv_loop_t = undefined; + if (c.uv_loop_init(&loop) != 0) return LibuvError.LoopInitFailed; + var loop_open = true; + errdefer { + if (loop_open) _ = c.uv_loop_close(&loop); + } + + var state = WorkState{}; + var work_req: c.uv_work_t = undefined; + work_req.data = @ptrCast(&state); + const queue_status = c.uv_queue_work(&loop, &work_req, workExecute, workAfter); + if (queue_status != 0) return LibuvError.WorkQueueFailed; + log.info("uv_queue_work submitted", .{}); + + const run_status = c.uv_run(&loop, c.UV_RUN_DEFAULT); + if (run_status != 0) return LibuvError.RunFailed; + + if (!state.executed) return LibuvError.WorkDidNotRun; + if (!state.after_called or state.after_status != 0) return LibuvError.WorkCompletionFailed; + + const close_status = c.uv_loop_close(&loop); + loop_open = false; + if (close_status != 0) return LibuvError.LoopCloseFailed; + log.info("Work queue scenario complete", .{}); +} + +pub fn main() LibuvError!void { + for (scenarios) |scenario| { + log.info("=== {s} ===", .{scenario.name}); + try scenario.run(); + } + log.info("All libuv smoke scenarios succeeded", .{}); +} diff --git a/tests/unit/reactor_effectors.zig b/tests/unit/reactor_effectors.zig new file mode 100644 index 0000000..7dc6ce4 --- /dev/null +++ b/tests/unit/reactor_effectors.zig @@ -0,0 +1,69 @@ +const std = @import("std"); +const zerver = @import("zerver"); + +const EffectDispatcher = zerver.reactor_effectors.EffectDispatcher; +const DispatchError = zerver.reactor_effectors.DispatchError; +const Context = zerver.reactor_effectors.Context; +const JobSystem = zerver.reactor_job_system.JobSystem; + +fn makeContext(loop: *zerver.libuv_reactor.Loop, jobs: *JobSystem) Context { + return Context{ .loop = loop, .jobs = jobs }; +} + +test "default handlers report unsupported" { + var loop = try zerver.libuv_reactor.Loop.init(); + defer loop.deinit() catch {}; + + var jobs: JobSystem = undefined; + try jobs.init(.{ .allocator = std.testing.allocator, .worker_count = 0 }); + defer jobs.deinit(); + + var dispatcher = EffectDispatcher.init(); + var ctx = makeContext(&loop, &jobs); + + const effects = [_]zerver.types.Effect{ + .{ .http_get = .{ .url = "http://example.com", .token = 0 } }, + .{ .http_head = .{ .url = "http://example.com/head", .token = 1 } }, + .{ .http_post = .{ .url = "http://example.com", .body = "{}", .token = 2 } }, + .{ .http_put = .{ .url = "http://example.com", .body = "{}", .token = 3 } }, + .{ .http_delete = .{ .url = "http://example.com", .token = 4 } }, + .{ .http_options = .{ .url = "http://example.com", .token = 5 } }, + .{ .http_trace = .{ .url = "http://example.com", .token = 6 } }, + .{ .http_connect = .{ .url = "http://example.com", .token = 7 } }, + .{ .http_patch = .{ .url = "http://example.com", .body = "{}", .token = 8 } }, + }; + + for (effects) |effect| { + try std.testing.expectError(DispatchError.UnsupportedEffect, dispatcher.dispatch(&ctx, effect)); + } +} + +test "custom handler executes" { + var loop = try zerver.libuv_reactor.Loop.init(); + defer loop.deinit() catch {}; + + var jobs: JobSystem = undefined; + try jobs.init(.{ .allocator = std.testing.allocator, .worker_count = 0 }); + defer jobs.deinit(); + + var dispatcher = EffectDispatcher.init(); + dispatcher.setHttpGetHandler(httpGetNoop); + + var ctx = makeContext(&loop, &jobs); + const supported = zerver.types.Effect{ .http_get = .{ + .url = "http://example.com", + .token = 0, + } }; + try dispatcher.dispatch(&ctx, supported); + + const unsupported = zerver.types.Effect{ .http_post = .{ + .url = "http://example.com", + .body = "{}", + .token = 1, + } }; + try std.testing.expectError(DispatchError.UnsupportedEffect, dispatcher.dispatch(&ctx, unsupported)); +} + +fn httpGetNoop(_: *Context, payload: zerver.types.HttpGet) DispatchError!void { + std.debug.assert(std.mem.eql(u8, payload.url, "http://example.com")); +} diff --git a/tests/unit/reactor_job_system.zig b/tests/unit/reactor_job_system.zig new file mode 100644 index 0000000..bc4c787 --- /dev/null +++ b/tests/unit/reactor_job_system.zig @@ -0,0 +1,56 @@ +const std = @import("std"); +const zerver = @import("zerver"); + +const JobSystem = zerver.reactor_job_system.JobSystem; +const SubmitError = zerver.reactor_job_system.SubmitError; + +const Counter = struct { + value: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), +}; + +fn incrementJob(ctx: *anyopaque) void { + const counter: *Counter = @ptrCast(@alignCast(ctx)); + _ = counter.value.fetchAdd(1, .seq_cst); +} + +test "job system executes submitted jobs" { + var js: JobSystem = undefined; + try js.init(.{ .allocator = std.testing.allocator, .worker_count = 2 }); + defer js.deinit(); + + var counter = Counter{}; + const total: u32 = 8; + + var i: u32 = 0; + while (i < total) : (i += 1) { + try js.submit(.{ .callback = incrementJob, .ctx = &counter }); + } + + var attempt: usize = 0; + while (counter.value.load(.seq_cst) < total and attempt < 10_000) : (attempt += 1) { + std.Thread.sleep(1_000_000); // 1 ms + } + + try std.testing.expectEqual(total, counter.value.load(.seq_cst)); +} + +test "job system rejects submissions after shutdown" { + var js: JobSystem = undefined; + try js.init(.{ .allocator = std.testing.allocator, .worker_count = 1 }); + defer js.deinit(); + + js.shutdown(); + + var counter = Counter{}; + try std.testing.expectError(SubmitError.ShuttingDown, js.submit(.{ .callback = incrementJob, .ctx = &counter })); +} + +test "job system enforces queue capacity" { + var js: JobSystem = undefined; + try js.init(.{ .allocator = std.testing.allocator, .worker_count = 0, .queue_capacity = 1 }); + defer js.deinit(); + + var counter = Counter{}; + try js.submit(.{ .callback = incrementJob, .ctx = &counter }); + try std.testing.expectError(SubmitError.QueueFull, js.submit(.{ .callback = incrementJob, .ctx = &counter })); +} diff --git a/tests/unit/reactor_join.zig b/tests/unit/reactor_join.zig new file mode 100644 index 0000000..6b46fb0 --- /dev/null +++ b/tests/unit/reactor_join.zig @@ -0,0 +1,81 @@ +const std = @import("std"); +const join = @import("zerver").reactor_join; + +const Mode = join.Mode; +const Join = join.Join; + +fn cfg(join_kind: Join) join.JoinConfig { + return .{ .mode = Mode.Parallel, .join = join_kind }; +} + +fn makeState(join_kind: Join, total: usize, required: usize) join.JoinState { + return join.JoinState.init(cfg(join_kind), total, required); +} + +test "join all succeeds after all completions" { + var state = makeState(.all, 2, 1); + try std.testing.expect(state.record(.{ .required = false, .success = true }) == .Pending); + const resolution = state.record(.{ .required = true, .success = true }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.success, resolution.Resume.status); +} + +test "join all fails on required failure" { + var state = makeState(.all, 2, 1); + const resolution = state.record(.{ .required = true, .success = false }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.failure, resolution.Resume.status); +} + +test "join any resumes on first completion" { + var state = makeState(.any, 2, 1); + const resolution = state.record(.{ .required = false, .success = false }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.success, resolution.Resume.status); +} + +test "join any propagates required failure" { + var state = makeState(.any, 2, 1); + const resolution = state.record(.{ .required = true, .success = false }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.failure, resolution.Resume.status); +} + +test "join first_success resumes on first success" { + var state = makeState(.first_success, 3, 1); + try std.testing.expect(state.record(.{ .required = false, .success = false }) == .Pending); + const resolution = state.record(.{ .required = false, .success = true }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.success, resolution.Resume.status); +} + +test "join first_success fails when required fail and no success" { + var state = makeState(.first_success, 2, 1); + try std.testing.expect(state.record(.{ .required = false, .success = false }) == .Pending); + const resolution = state.record(.{ .required = true, .success = false }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.failure, resolution.Resume.status); +} + +test "join first_success succeeds when only optional failures" { + var state = makeState(.first_success, 2, 0); + try std.testing.expect(state.record(.{ .required = false, .success = false }) == .Pending); + const resolution = state.record(.{ .required = false, .success = false }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.success, resolution.Resume.status); +} + +test "join all_required resumes when required complete" { + var state = makeState(.all_required, 3, 2); + try std.testing.expect(state.record(.{ .required = true, .success = true }) == .Pending); + const resolution = state.record(.{ .required = true, .success = true }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.success, resolution.Resume.status); +} + +test "join all_required fails on required failure" { + var state = makeState(.all_required, 2, 2); + const resolution = state.record(.{ .required = true, .success = false }); + try std.testing.expect(resolution == .Resume); + try std.testing.expectEqual(join.Status.failure, resolution.Resume.status); +} diff --git a/tests/unit/reactor_saga.zig b/tests/unit/reactor_saga.zig new file mode 100644 index 0000000..c6d71fa --- /dev/null +++ b/tests/unit/reactor_saga.zig @@ -0,0 +1,24 @@ +const std = @import("std"); +const zerver = @import("zerver"); + +const SagaLog = zerver.reactor_saga.SagaLog; + +test "saga log stub reports unimplemented" { + var log = SagaLog.init(std.testing.allocator); + defer log.deinit(); + + try std.testing.expectEqual(@as(usize, 0), log.len()); + + const compensation: zerver.types.Compensation = .{ + .label = "stub", + .effect = .{ .http_get = .{ + .url = "http://example.com", + .token = 1, + } }, + }; + + try std.testing.expectError(zerver.reactor_saga.SagaError.Unimplemented, log.record(compensation)); + try std.testing.expectEqual(@as(usize, 0), log.len()); + try std.testing.expect(log.pop() == null); + log.clear(); +} diff --git a/tests/unit/reactor_task_system.zig b/tests/unit/reactor_task_system.zig new file mode 100644 index 0000000..6605c67 --- /dev/null +++ b/tests/unit/reactor_task_system.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const zerver = @import("zerver"); + +const TaskSystem = zerver.reactor_task_system.TaskSystem; +const TaskSystemConfig = zerver.reactor_task_system.TaskSystemConfig; +const TaskSystemError = zerver.reactor_task_system.TaskSystemError; +const ComputePoolKind = zerver.reactor_task_system.ComputePoolKind; +const Job = zerver.reactor_job_system.Job; + +const Counter = struct { + value: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), +}; + +fn incrementJob(ctx: *anyopaque) void { + const counter: *Counter = @ptrCast(@alignCast(ctx)); + _ = counter.value.fetchAdd(1, .seq_cst); +} + +test "task system runs continuation jobs" { + var ts: TaskSystem = undefined; + try ts.init(.{ + .allocator = std.testing.allocator, + .continuation_workers = 2, + }); + defer ts.deinit(); + + var counter = Counter{}; + const total: u32 = 6; + + var i: u32 = 0; + while (i < total) : (i += 1) { + try ts.submitContinuation(Job{ .callback = incrementJob, .ctx = &counter }); + } + + var attempt: usize = 0; + while (counter.value.load(.seq_cst) < total and attempt < 10_000) : (attempt += 1) { + std.Thread.sleep(1_000_000); + } + + try std.testing.expectEqual(total, counter.value.load(.seq_cst)); +} + +test "task system runs compute jobs" { + var ts: TaskSystem = undefined; + try ts.init(.{ + .allocator = std.testing.allocator, + .continuation_workers = 1, + .compute_kind = ComputePoolKind.dedicated, + .compute_workers = 1, + }); + defer ts.deinit(); + + var counter = Counter{}; + try ts.submitCompute(Job{ .callback = incrementJob, .ctx = &counter }); + + var attempt: usize = 0; + while (counter.value.load(.seq_cst) < 1 and attempt < 10_000) : (attempt += 1) { + std.Thread.sleep(1_000_000); + } + + try std.testing.expectEqual(@as(u32, 1), counter.value.load(.seq_cst)); +} + +test "task system errors when compute pool disabled" { + var ts: TaskSystem = undefined; + try ts.init(.{ + .allocator = std.testing.allocator, + .continuation_workers = 1, + }); + defer ts.deinit(); + + var counter = Counter{}; + try std.testing.expectError(TaskSystemError.NoComputePool, ts.submitCompute(Job{ .callback = incrementJob, .ctx = &counter })); +} + +test "task system shared compute uses continuation pool" { + var ts: TaskSystem = undefined; + try ts.init(.{ + .allocator = std.testing.allocator, + .continuation_workers = 1, + .compute_kind = ComputePoolKind.shared, + }); + defer ts.deinit(); + + var counter = Counter{}; + try ts.submitCompute(Job{ .callback = incrementJob, .ctx = &counter }); + + var attempt: usize = 0; + while (counter.value.load(.seq_cst) < 1 and attempt < 10_000) : (attempt += 1) { + std.Thread.sleep(1_000_000); + } + + try std.testing.expectEqual(@as(u32, 1), counter.value.load(.seq_cst)); + try std.testing.expect(ts.hasComputePool()); + const shared_jobs = ts.computeJobs() orelse unreachable; + try std.testing.expectEqual(@intFromPtr(shared_jobs), @intFromPtr(ts.continuationJobs())); +} diff --git a/third_party/libuv b/third_party/libuv new file mode 160000 index 0000000..a944c42 --- /dev/null +++ b/third_party/libuv @@ -0,0 +1 @@ +Subproject commit a944c422cca5522073e03710ca7fd08f53218358 From f8ed3226d5c9226e6257728fcc3025ef8255b7b4 Mon Sep 17 00:00:00 2001 From: Earl Cameron Date: Sun, 26 Oct 2025 14:30:10 -0700 Subject: [PATCH 2/4] feat: Introduce Scheduler and RuntimeEngine with enhanced resource management and error handling --- src/zerver/impure/executor.zig | 18 ++- src/zerver/observability/slog.zig | 14 +- src/zerver/runtime/reactor/libuv.zig | 187 +++++++++++++++++++++++++- src/zerver/runtime/resources.zig | 106 ++++++++++++--- src/zerver/runtime/runtime_engine.zig | 43 ++++++ src/zerver/runtime/scheduler.zig | 80 +++++++++++ 6 files changed, 415 insertions(+), 33 deletions(-) create mode 100644 src/zerver/runtime/runtime_engine.zig create mode 100644 src/zerver/runtime/scheduler.zig diff --git a/src/zerver/impure/executor.zig b/src/zerver/impure/executor.zig index 35982a2..2b88b49 100644 --- a/src/zerver/impure/executor.zig +++ b/src/zerver/impure/executor.zig @@ -28,7 +28,7 @@ pub const ExecutionMode = enum { const ReactorNeedRunner = struct { const Completion = struct { - result: types.EffectResult, + result: types.EffectResult, required: bool, effect: *const types.Effect, sequence: usize, @@ -573,7 +573,7 @@ pub const Executor = struct { pub fn init( allocator: std.mem.Allocator, - effect_handler: *const fn (*const types.Effect, u32) anyerror!types.EffectResult, + effect_handler: *const fn (*const types.Effect, u32) anyerror!types.EffectResult, ) Executor { return .{ .allocator = allocator, @@ -663,7 +663,7 @@ pub const Executor = struct { } // Track effect results by token (slot identifier) - var results = std.AutoHashMap(u32, types.EffectResult).init(ctx_base.allocator); + var results = std.AutoHashMap(u32, types.EffectResult).init(ctx_base.allocator); defer results.deinit(); const total_effects = need.effects.len; @@ -985,15 +985,23 @@ fn requiresComputePool(effect: types.Effect) bool { } fn effectQueueFailure(effect: types.Effect, err: anyerror) types.Error { + const kind: u16 = switch (err) { + reactor_jobs.SubmitError.QueueFull => types.ErrorCode.TooManyRequests, + else => types.ErrorCode.UpstreamUnavailable, + }; return .{ - .kind = types.ErrorCode.UpstreamUnavailable, + .kind = kind, .ctx = .{ .what = @tagName(effect), .key = @errorName(err) }, }; } fn continuationQueueFailure(err: anyerror) types.Error { + const kind: u16 = switch (err) { + reactor_jobs.SubmitError.QueueFull => types.ErrorCode.TooManyRequests, + else => types.ErrorCode.UpstreamUnavailable, + }; return .{ - .kind = types.ErrorCode.UpstreamUnavailable, + .kind = kind, .ctx = .{ .what = "continuation", .key = @errorName(err) }, }; } diff --git a/src/zerver/observability/slog.zig b/src/zerver/observability/slog.zig index 6dbed4f..d7f71c8 100644 --- a/src/zerver/observability/slog.zig +++ b/src/zerver/observability/slog.zig @@ -84,12 +84,14 @@ pub const Attr = struct { } pub fn enumeration(key: []const u8, value: anytype) Attr { - const T = @TypeOf(value); - return switch (@typeInfo(T)) { - .Enum => .{ .key = key, .value = .{ .enum_tag = @tagName(value) } }, - .Union => |union_info| switch (union_info.tag_type) { - null => @compileError("Attr.enumeration expects a tagged union"), - else => .{ .key = key, .value = .{ .enum_tag = @tagName(value) } }, + const type_info = @typeInfo(@TypeOf(value)); + return switch (type_info) { + .@"enum" => .{ .key = key, .value = .{ .enum_tag = @tagName(value) } }, + .@"union" => |union_info| blk: { + if (union_info.tag_type == null) { + @compileError("Attr.enumeration expects a tagged union"); + } + break :blk .{ .key = key, .value = .{ .enum_tag = @tagName(value) } }; }, else => @compileError("Attr.enumeration expects an enum or tagged union"), }; diff --git a/src/zerver/runtime/reactor/libuv.zig b/src/zerver/runtime/reactor/libuv.zig index e93d442..9213e1e 100644 --- a/src/zerver/runtime/reactor/libuv.zig +++ b/src/zerver/runtime/reactor/libuv.zig @@ -7,6 +7,10 @@ const c = @cImport({ pub const Error = error{ LoopInitFailed, LoopCloseFailed, + AsyncInitFailed, + AsyncSendFailed, + TimerInitFailed, + WorkSubmitFailed, }; pub const RunMode = enum { @@ -34,10 +38,10 @@ pub const Loop = struct { } pub fn run(self: *Loop, mode: RunMode) bool { - const uv_mode = switch (mode) { - .default => c.UV_RUN_DEFAULT, - .once => c.UV_RUN_ONCE, - .nowait => c.UV_RUN_NOWAIT, + const uv_mode: c.uv_run_mode = switch (mode) { + .default => @as(c.uv_run_mode, c.UV_RUN_DEFAULT), + .once => @as(c.uv_run_mode, c.UV_RUN_ONCE), + .nowait => @as(c.uv_run_mode, c.UV_RUN_NOWAIT), }; const result = c.uv_run(&self.inner, uv_mode); return result != 0; @@ -51,3 +55,178 @@ pub const Loop = struct { return &self.inner; } }; + +pub const Async = struct { + handle: c.uv_async_t = undefined, + callback: Callback = defaultCallback, + user_data: ?*anyopaque = null, + initialized: bool = false, + + pub const Callback = *const fn (*Async) void; + + pub fn init(self: *Async, loop: *Loop, callback: Callback, user_data: ?*anyopaque) Error!void { + self.* = .{ + .handle = undefined, + .callback = callback, + .user_data = user_data, + .initialized = false, + }; + + const rc = c.uv_async_init(loop.ptr(), &self.handle, asyncTrampoline); + if (rc != 0) return Error.AsyncInitFailed; + + self.handle.data = self; + self.initialized = true; + } + + pub fn close(self: *Async) void { + if (!self.initialized) return; + c.uv_close(@ptrCast(@alignCast(&self.handle)), asyncCloseCallback); + self.initialized = false; + } + + pub fn send(self: *Async) Error!void { + if (!self.initialized) return; + const rc = c.uv_async_send(&self.handle); + if (rc != 0) return Error.AsyncSendFailed; + } + + pub fn setUserData(self: *Async, data: ?*anyopaque) void { + self.user_data = data; + } + + pub fn getUserData(self: *Async) ?*anyopaque { + return self.user_data; + } + + fn asyncTrampoline(handle: [*c]c.uv_async_t) callconv(.c) void { + const async_ptr = handle_to_async(handle); + async_ptr.callback(async_ptr); + } + + fn defaultCallback(_: *Async) void {} +}; + +pub const Timer = struct { + handle: c.uv_timer_t = undefined, + callback: Callback = defaultCallback, + user_data: ?*anyopaque = null, + initialized: bool = false, + + pub const Callback = *const fn (*Timer) void; + + pub fn init(self: *Timer, loop: *Loop, callback: Callback, user_data: ?*anyopaque) Error!void { + self.* = .{ + .handle = undefined, + .callback = callback, + .user_data = user_data, + .initialized = false, + }; + + const rc = c.uv_timer_init(loop.ptr(), &self.handle); + if (rc != 0) return Error.TimerInitFailed; + + self.handle.data = self; + self.initialized = true; + } + + pub fn start(self: *Timer, timeout_ms: u64, repeat_ms: u64) Error!void { + if (!self.initialized) return; + const rc = c.uv_timer_start(&self.handle, timerTrampoline, timeout_ms, repeat_ms); + if (rc != 0) return Error.TimerInitFailed; + } + + pub fn stop(self: *Timer) void { + if (!self.initialized) return; + _ = c.uv_timer_stop(&self.handle); + } + + pub fn close(self: *Timer) void { + if (!self.initialized) return; + c.uv_close(@ptrCast(@alignCast(&self.handle)), timerCloseCallback); + self.initialized = false; + } + + pub fn setUserData(self: *Timer, data: ?*anyopaque) void { + self.user_data = data; + } + + pub fn getUserData(self: *Timer) ?*anyopaque { + return self.user_data; + } + + fn timerTrampoline(handle: [*c]c.uv_timer_t) callconv(.c) void { + const timer_ptr = handle_to_timer(handle); + timer_ptr.callback(timer_ptr); + } + + fn defaultCallback(_: *Timer) void {} +}; + +pub const Work = struct { + request: c.uv_work_t = undefined, + work_cb: WorkCallback = defaultWork, + after_cb: AfterWorkCallback = defaultAfterWork, + user_data: ?*anyopaque = null, + submitted: bool = false, + + pub const WorkCallback = *const fn (*Work) void; + pub const AfterWorkCallback = *const fn (*Work, c_int) void; + + pub fn submit(self: *Work, loop: *Loop, work_cb: WorkCallback, after_cb: AfterWorkCallback, user_data: ?*anyopaque) Error!void { + self.* = .{ + .request = undefined, + .work_cb = work_cb, + .after_cb = after_cb, + .user_data = user_data, + .submitted = false, + }; + + self.request.data = self; + const rc = c.uv_queue_work(loop.ptr(), &self.request, workTrampoline, afterWorkTrampoline); + if (rc != 0) return Error.WorkSubmitFailed; + + self.submitted = true; + } + + pub fn getUserData(self: *Work) ?*anyopaque { + return self.user_data; + } + + fn workTrampoline(req: [*c]c.uv_work_t) callconv(.c) void { + const work_ptr = request_to_work(req); + work_ptr.work_cb(work_ptr); + } + + fn afterWorkTrampoline(req: [*c]c.uv_work_t, status: c_int) callconv(.c) void { + const work_ptr = request_to_work(req); + work_ptr.after_cb(work_ptr, status); + work_ptr.submitted = false; + } + + fn defaultWork(_: *Work) void {} + fn defaultAfterWork(_: *Work, _: c_int) void {} +}; + +fn handle_to_async(handle: [*c]c.uv_async_t) *Async { + const raw = handle.*.data orelse unreachable; + return @ptrCast(@alignCast(raw)); +} + +fn handle_to_timer(handle: [*c]c.uv_timer_t) *Timer { + const raw = handle.*.data orelse unreachable; + return @ptrCast(@alignCast(raw)); +} + +fn request_to_work(req: [*c]c.uv_work_t) *Work { + const raw = req.*.data orelse unreachable; + return @ptrCast(@alignCast(raw)); +} + +fn asyncCloseCallback(handle: [*c]c.uv_handle_t) callconv(.c) void { + _ = handle; +} + +fn timerCloseCallback(handle: [*c]c.uv_handle_t) callconv(.c) void { + _ = handle; +} diff --git a/src/zerver/runtime/resources.zig b/src/zerver/runtime/resources.zig index e21c180..c851bd9 100644 --- a/src/zerver/runtime/resources.zig +++ b/src/zerver/runtime/resources.zig @@ -5,29 +5,40 @@ const task_system = @import("reactor/task_system.zig"); const job_system = @import("reactor/job_system.zig"); const effectors = @import("reactor/effectors.zig"); const libuv = @import("reactor/libuv.zig"); +const scheduler_mod = @import("scheduler.zig"); + +const AtomicOrder = std.builtin.AtomicOrder; const sqlite_driver = &sql.dialects.sqlite.driver.driver; const ReactorResources = struct { enabled: bool = false, - task_system: task_system.TaskSystem = undefined, + scheduler: scheduler_mod.Scheduler = .{}, effector_jobs: job_system.JobSystem = undefined, - has_task_system: bool = false, + has_scheduler: bool = false, has_effector_jobs: bool = false, dispatcher: effectors.EffectDispatcher = effectors.EffectDispatcher.init(), loop: libuv.Loop = undefined, loop_initialized: bool = false, + loop_thread: ?std.Thread = null, + loop_should_run: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + wake_handle: libuv.Async = undefined, + wake_initialized: bool = false, fn init(self: *ReactorResources, allocator: std.mem.Allocator, cfg: config_mod.ReactorConfig) !void { self.* = .{ .enabled = cfg.enabled, - .has_task_system = false, + .has_scheduler = false, .has_effector_jobs = false, - .task_system = undefined, + .scheduler = .{}, .effector_jobs = undefined, .dispatcher = effectors.EffectDispatcher.init(), .loop = undefined, .loop_initialized = false, + .loop_thread = null, + .loop_should_run = std.atomic.Value(bool).init(false), + .wake_handle = undefined, + .wake_initialized = false, }; if (!cfg.enabled) return; @@ -45,30 +56,60 @@ const ReactorResources = struct { }); self.has_effector_jobs = true; - try self.task_system.init(.{ + try self.scheduler.init(.{ .allocator = allocator, .continuation_workers = cfg.continuation_pool.size, .continuation_queue_capacity = cfg.continuation_pool.queue_capacity, .compute_kind = convertComputeKind(cfg.compute_pool.kind), .compute_workers = cfg.compute_pool.size, .compute_queue_capacity = cfg.compute_pool.queue_capacity, + .label = "scheduler", }); - self.has_task_system = true; + self.has_scheduler = true; + + try self.wake_handle.init(&self.loop, wakeCallback, self); + self.wake_initialized = true; + + self.loop_should_run.store(true, AtomicOrder.seq_cst); + self.loop_thread = try std.Thread.spawn(.{}, loopThreadMain, .{self}); } fn shutdown(self: *ReactorResources) void { if (!self.enabled) return; - if (self.loop_initialized) self.loop.stop(); - if (self.has_task_system) self.task_system.shutdown(); + if (self.loop_initialized) { + self.loop_should_run.store(false, AtomicOrder.seq_cst); + self.loop.stop(); + if (self.wake_initialized) { + self.triggerWake(); + } + if (self.loop_thread) |thread| { + thread.join(); + self.loop_thread = null; + } else { + while (self.loop.run(.nowait)) {} + } + if (self.wake_initialized) { + self.wake_handle.close(); + self.wake_initialized = false; + while (self.loop.run(.nowait)) {} + } + } else { + self.loop_should_run.store(false, AtomicOrder.seq_cst); + if (self.wake_initialized) { + self.wake_handle.close(); + self.wake_initialized = false; + } + } + if (self.has_scheduler) self.scheduler.shutdown(); if (self.has_effector_jobs) self.effector_jobs.shutdown(); } fn deinit(self: *ReactorResources) void { if (!self.enabled) return; self.shutdown(); - if (self.has_task_system) { - self.task_system.deinit(); - self.has_task_system = false; + if (self.has_scheduler) { + self.scheduler.deinit(); + self.has_scheduler = false; } if (self.has_effector_jobs) { self.effector_jobs.deinit(); @@ -80,15 +121,19 @@ const ReactorResources = struct { } self.dispatcher = effectors.EffectDispatcher.init(); self.enabled = false; + self.loop_thread = null; + self.loop_should_run = std.atomic.Value(bool).init(false); } fn taskSystem(self: *ReactorResources) ?*task_system.TaskSystem { - if (!self.enabled or !self.has_task_system) return null; - return &self.task_system; + if (!self.enabled) return null; + if (!self.has_scheduler) return null; + return self.scheduler.taskSystem(); } fn effectorJobs(self: *ReactorResources) ?*job_system.JobSystem { - if (!self.enabled or !self.has_effector_jobs) return null; + if (!self.enabled) return null; + if (!self.has_effector_jobs) return null; return &self.effector_jobs; } @@ -98,23 +143,31 @@ const ReactorResources = struct { } fn loopPtr(self: *ReactorResources) ?*libuv.Loop { - if (!self.enabled or !self.loop_initialized) return null; + if (!self.enabled) return null; + if (!self.loop_initialized) return null; return &self.loop; } fn context(self: *ReactorResources) ?effectors.Context { - if (!self.enabled or !self.has_effector_jobs or !self.loop_initialized) return null; - const compute_jobs = if (self.has_task_system) self.task_system.computeJobs() else null; + if (!self.enabled) return null; + if (!self.has_effector_jobs) return null; + if (!self.loop_initialized) return null; + const compute_jobs = if (self.has_scheduler) self.scheduler.computeJobs() else null; return effectors.Context{ .loop = &self.loop, .jobs = &self.effector_jobs, .compute_jobs = compute_jobs, .accelerator_jobs = null, .kv_cache = null, - .task_system = if (self.has_task_system) &self.task_system else null, + .task_system = if (self.has_scheduler) self.scheduler.taskSystem() else null, }; } + fn triggerWake(self: *ReactorResources) void { + if (!self.wake_initialized) return; + self.wake_handle.send() catch {}; + } + fn convertComputeKind(kind: config_mod.ComputePoolKind) task_system.ComputePoolKind { return switch (kind) { .disabled => .disabled, @@ -124,6 +177,23 @@ const ReactorResources = struct { } }; +fn loopThreadMain(self: *ReactorResources) void { + while (self.loop_should_run.load(AtomicOrder.seq_cst)) { + const active = self.loop.run(.once); + if (!active) { + std.Thread.sleep(1 * std.time.ns_per_ms); + } + } + + while (self.loop.run(.nowait)) {} +} + +fn wakeCallback(async_handle: *libuv.Async) void { + const raw = async_handle.getUserData() orelse return; + const resources: *ReactorResources = @ptrCast(@alignCast(raw)); + _ = resources; +} + pub const RuntimeResources = struct { allocator: std.mem.Allocator, config: config_mod.AppConfig, diff --git a/src/zerver/runtime/runtime_engine.zig b/src/zerver/runtime/runtime_engine.zig new file mode 100644 index 0000000..b1ef49e --- /dev/null +++ b/src/zerver/runtime/runtime_engine.zig @@ -0,0 +1,43 @@ +const std = @import("std"); +const slog = @import("../observability/slog.zig"); +const config_mod = @import("config.zig"); +const resources_mod = @import("resources.zig"); +const runtime_global = @import("global.zig"); + +pub const RuntimeEngine = struct { + allocator: std.mem.Allocator, + resources_ptr: ?*resources_mod.RuntimeResources = null, + + pub fn init(allocator: std.mem.Allocator, config: config_mod.AppConfig) !RuntimeEngine { + var cfg = config; + const res = resources_mod.create(allocator, cfg) catch |err| { + cfg.deinit(allocator); + return err; + }; + + runtime_global.set(res); + slog.debug("runtime_engine_started", &.{ + slog.Attr.bool("reactor_enabled", res.reactorEnabled()), + }); + + return .{ + .allocator = allocator, + .resources_ptr = res, + }; + } + + pub fn resources(self: *RuntimeEngine) *resources_mod.RuntimeResources { + return self.resources_ptr orelse @panic("runtime engine not initialized"); + } + + pub fn shutdown(self: *RuntimeEngine) void { + const res = self.resources_ptr orelse return; + slog.debug("runtime_engine_shutdown", &.{ + slog.Attr.bool("reactor_enabled", res.reactorEnabled()), + }); + res.deinit(); + self.allocator.destroy(res); + runtime_global.clear(); + self.resources_ptr = null; + } +}; diff --git a/src/zerver/runtime/scheduler.zig b/src/zerver/runtime/scheduler.zig new file mode 100644 index 0000000..fc78fda --- /dev/null +++ b/src/zerver/runtime/scheduler.zig @@ -0,0 +1,80 @@ +const std = @import("std"); +const task_system = @import("reactor/task_system.zig"); +const job_system = @import("reactor/job_system.zig"); +const slog = @import("../observability/slog.zig"); + +pub const SchedulerConfig = struct { + allocator: std.mem.Allocator, + continuation_workers: usize, + continuation_queue_capacity: usize = 0, + compute_kind: task_system.ComputePoolKind = .disabled, + compute_workers: usize = 0, + compute_queue_capacity: usize = 0, + label: []const u8 = "scheduler", +}; + +pub const Scheduler = struct { + inner: task_system.TaskSystem = undefined, + initialized: bool = false, + label: []const u8 = "scheduler", + + pub fn init(self: *Scheduler, cfg: SchedulerConfig) !void { + self.label = cfg.label; + try self.inner.init(.{ + .allocator = cfg.allocator, + .continuation_workers = cfg.continuation_workers, + .continuation_queue_capacity = cfg.continuation_queue_capacity, + .compute_kind = cfg.compute_kind, + .compute_workers = cfg.compute_workers, + .compute_queue_capacity = cfg.compute_queue_capacity, + }); + self.initialized = true; + slog.debug("scheduler_init", &.{ + slog.Attr.string("label", self.label), + slog.Attr.uint("continuation_workers", @as(u64, @intCast(cfg.continuation_workers))), + slog.Attr.string("compute_kind", @tagName(cfg.compute_kind)), + slog.Attr.uint("compute_workers", @as(u64, @intCast(cfg.compute_workers))), + }); + } + + pub fn shutdown(self: *Scheduler) void { + if (!self.initialized) return; + slog.debug("scheduler_shutdown", &.{ + slog.Attr.string("label", self.label), + }); + self.inner.shutdown(); + } + + pub fn deinit(self: *Scheduler) void { + if (!self.initialized) return; + slog.debug("scheduler_deinit", &.{ + slog.Attr.string("label", self.label), + }); + self.inner.deinit(); + self.initialized = false; + } + + pub fn submitContinuation(self: *Scheduler, job: job_system.Job) task_system.TaskSystemError!void { + return self.inner.submitContinuation(job); + } + + pub fn submitCompute(self: *Scheduler, job: job_system.Job) task_system.TaskSystemError!void { + return self.inner.submitCompute(job); + } + + pub fn continuationJobs(self: *Scheduler) *job_system.JobSystem { + return self.inner.continuationJobs(); + } + + pub fn computeJobs(self: *Scheduler) ?*job_system.JobSystem { + return self.inner.computeJobs(); + } + + pub fn hasComputePool(self: *Scheduler) bool { + return self.inner.hasComputePool(); + } + + pub fn taskSystem(self: *Scheduler) *task_system.TaskSystem { + return &self.inner; + } +}; From e5981fa0b1e27127467674d8a5232681a86d22ef Mon Sep 17 00:00:00 2001 From: Earl Cameron Date: Sun, 26 Oct 2025 21:56:08 -0700 Subject: [PATCH 3/4] Refactor telemetry events and tracer for job system enhancements - Renamed `ContinuationEvent` to `StepResumeEvent` and updated related documentation. - Introduced new telemetry events for effect and step job lifecycle: `EffectJobEnqueuedEvent`, `EffectJobStartedEvent`, `EffectJobCompletedEvent`, `StepJobEnqueuedEvent`, `StepJobStartedEvent`, `StepJobCompletedEvent`, and `StepWaitEvent`. - Updated the `Event` union to include new telemetry events. - Implemented corresponding logging and emission methods in the `Telemetry` struct for the new events. - Enhanced the `Tracer` struct to record new job-related events and updated the event kind enum. - Modified the job system to support step jobs instead of continuation jobs, including renaming methods and updating logging messages. - Added a new `termination.zig` file for cross-platform termination signal handling, ensuring graceful shutdown logging. --- main.zig | 10 +- src/zerver/bootstrap/init.zig | 2 +- src/zerver/core/core.zig | 119 +++-- src/zerver/core/ctx.zig | 21 +- src/zerver/impure/executor.zig | 185 ++++++-- src/zerver/impure/server.zig | 172 ++++++- src/zerver/observability/otel.zig | 518 ++++++++++++++++++++- src/zerver/observability/telemetry.zig | 295 +++++++++++- src/zerver/observability/tracer.zig | 160 ++++++- src/zerver/runtime/handler.zig | 1 - src/zerver/runtime/reactor/job_system.zig | 26 ++ src/zerver/runtime/reactor/task_system.zig | 12 +- src/zerver/runtime/scheduler.zig | 8 +- src/zerver/runtime/termination.zig | 72 +++ 14 files changed, 1487 insertions(+), 114 deletions(-) create mode 100644 src/zerver/runtime/termination.zig diff --git a/main.zig b/main.zig index f897285..097f452 100644 --- a/main.zig +++ b/main.zig @@ -2,7 +2,7 @@ const std = @import("std"); const server_init = @import("src/zerver/bootstrap/init.zig"); const slog = @import("src/zerver/observability/slog.zig"); -const listener = @import("src/zerver/runtime/listener.zig"); +const termination = @import("src/zerver/runtime/termination.zig"); pub fn main() !void { try slog.setupDefaultLoggerWithFile("logs/server.log"); @@ -11,6 +11,12 @@ pub fn main() !void { slog.Attr.string("file", "logs/server.log"), }); + termination.installHandlers() catch |err| { + slog.warn("Failed to install termination handlers", &.{ + slog.Attr.string("error", @errorName(err)), + }); + }; + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); const allocator = gpa.allocator(); @@ -23,5 +29,5 @@ pub fn main() !void { server_init.printDemoInfo(init_bundle.resources.configPtr()); // Start listening and serving - try listener.listenAndServe(&init_bundle.server, allocator); + try init_bundle.server.listen(); } diff --git a/src/zerver/bootstrap/init.zig b/src/zerver/bootstrap/init.zig index dda7e10..f71e7f4 100644 --- a/src/zerver/bootstrap/init.zig +++ b/src/zerver/bootstrap/init.zig @@ -82,7 +82,7 @@ pub fn initializeServer(allocator: std.mem.Allocator) !Initialization { const reactor_cfg = app_config.reactor; slog.info("reactor_config", &[_]slog.Attr{ - slog.Attr.@"bool"("enabled", reactor_cfg.enabled), + slog.Attr.bool("enabled", reactor_cfg.enabled), slog.Attr.uint("continuation_workers", reactor_cfg.continuation_pool.size), slog.Attr.uint("continuation_queue", reactor_cfg.continuation_pool.queue_capacity), slog.Attr.uint("effector_workers", reactor_cfg.effector_pool.size), diff --git a/src/zerver/core/core.zig b/src/zerver/core/core.zig index 705d061..e1dc240 100644 --- a/src/zerver/core/core.zig +++ b/src/zerver/core/core.zig @@ -9,54 +9,111 @@ const ctx_module = @import("ctx.zig"); /// Expected function signature: fn (*CtxBase) !Decision /// Or: fn (*CtxView(spec)) !Decision (with spec containing reads/writes) pub fn step(comptime name: []const u8, comptime F: anytype) types.Step { - // For now, assume all functions take *CtxBase directly - // TODO: Support CtxView functions in the future - // TODO: Logical Error - The 'step' function currently sets 'reads' and 'writes' to empty arrays. This bypasses CtxView's compile-time access control. Implement a mechanism to extract 'reads' and 'writes' from the step function's CtxView specification. + const fn_type = switch (@typeInfo(@TypeOf(F))) { + .@"fn" => |info| info, + else => @compileError("step expects a function value"), + }; + + if (fn_type.params.len == 0 or fn_type.params[0].type == null) { + @compileError("step function must accept a context parameter"); + } + + const param_type = fn_type.params[0].type.?; + const param_info = switch (@typeInfo(param_type)) { + .pointer => |info| info, + else => @compileError("step function must accept a pointer parameter"), + }; + + const child_type = param_info.child; + + if (child_type == ctx_module.CtxBase) { + return types.Step{ + .name = name, + .call = F, + .reads = &.{}, + .writes = &.{}, + }; + } + + if (!isCtxViewType(child_type)) { + @compileError("step function parameter must be *CtxBase or *CtxView(...) type"); + } + + const metadata = extractReadsWrites(child_type); + const trampoline = makeTrampolineFor(F, param_type); + return types.Step{ .name = name, - .call = F, - .reads = &.{}, - .writes = &.{}, + .call = trampoline, + .reads = metadata.reads, + .writes = metadata.writes, }; } /// Extract reads and writes from a CtxView type by inspecting its public decls. fn extractReadsWrites(comptime _CtxViewType: type) struct { reads: []const u32, writes: []const u32 } { - // For now, return empty - will be populated by context when spec is available - // The Step struct can be annotated separately, or we could enhance CtxView - // to expose its spec via a public const field. - _ = _CtxViewType; + if (!@hasDecl(_CtxViewType, "__reads") or !@hasDecl(_CtxViewType, "__writes")) { + return .{ .reads = &.{}, .writes = &.{} }; + } + return .{ - .reads = &.{}, - .writes = &.{}, + .reads = convertSlotsToIds(_CtxViewType.__reads), + .writes = convertSlotsToIds(_CtxViewType.__writes), }; } /// Create a wrapper function that adapts from *CtxBase to the typed view expected by F. -fn makeTrampolineFor(comptime F: anytype, comptime CtxViewPtr: anytype) *const fn (*anyopaque) anyerror!types.Decision { +fn makeTrampolineFor(comptime F: anytype, comptime CtxViewPtr: type) *const fn (*ctx_module.CtxBase) anyerror!types.Decision { + const ptr_info = @typeInfo(CtxViewPtr); + if (ptr_info != .Pointer) { + @compileError("CtxView trampoline expects a pointer type"); + } + const CtxViewType = ptr_info.Pointer.child; + return struct { - pub fn wrapper(base: *anyopaque) anyerror!types.Decision { - // Cast from *anyopaque back to *CtxBase, asserting the alignment is correct - const ctx_base: *ctx_module.CtxBase = @ptrCast(@alignCast(base)); + pub fn wrapper(base: *ctx_module.CtxBase) anyerror!types.Decision { + var view = CtxViewType{ + .base = base, + }; + return F(&view); + } + }.wrapper; +} - // Create a CtxView instance - extract the type from the pointer - const CtxViewType = @typeInfo(CtxViewPtr).Pointer.child; +fn isCtxViewType(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .Struct) return false; - // Instantiate the CtxView with the base context - var view: CtxViewType = undefined; + inline for (info.Struct.fields) |field| { + if (std.mem.eql(u8, field.name, "base") and field.type == *ctx_module.CtxBase) { + return true; + } + } - // CtxView fields should be: base: *CtxBase - // We set it manually - if (@hasField(CtxViewType, "base")) { - @field(view, "base") = ctx_base; - } else { - @compileError("CtxView must have a 'base' field of type *CtxBase"); - } + return false; +} - // Call the typed function with the view - return F(&view); - } - }.wrapper; +fn convertSlotsToIds(comptime slots_ptr: anytype) []const u32 { + const ptr_info = @typeInfo(@TypeOf(slots_ptr)); + if (ptr_info != .Pointer) return &.{}; + + const child_type = ptr_info.Pointer.child; + const child_info = @typeInfo(child_type); + if (child_info != .Array) return &.{}; + + const slots_array = slots_ptr.*; + if (slots_array.len == 0) return &.{}; + + const ids_array = comptime buildIdArray(slots_array); + return ids_array[0..]; +} + +fn buildIdArray(comptime slots_array: anytype) [slots_array.len]u32 { + var ids = std.mem.zeroes([slots_array.len]u32); + inline for (slots_array, 0..) |slot, idx| { + ids[idx] = @intFromEnum(slot); + } + return ids; } /// Helper to create a Decision.Continue. diff --git a/src/zerver/core/ctx.zig b/src/zerver/core/ctx.zig index b4084e4..1757cfc 100644 --- a/src/zerver/core/ctx.zig +++ b/src/zerver/core/ctx.zig @@ -30,6 +30,7 @@ pub const CtxBase = struct { // Observability request_id: []const u8 = "", + user_sub: []const u8 = "", start_time: i64, // milliseconds status_code: u16 = 200, request_bytes: usize = 0, @@ -112,6 +113,10 @@ pub const CtxBase = struct { self.request_id = id; } + pub fn user(self: *CtxBase) []const u8 { + return self.user_sub; + } + pub fn status(self: *CtxBase) u16 { return self.status_code; } @@ -156,8 +161,18 @@ pub const CtxBase = struct { } pub fn setUser(self: *CtxBase, sub: []const u8) void { - _ = self.allocator.dupe(u8, sub) catch return; - // TODO: store user sub somewhere + const duped = self.allocator.dupe(u8, sub) catch return; + self.user_sub = duped; + } + + pub fn runExitCallbacks(self: *CtxBase) void { + var i = self.exit_cbs.items.len; + while (i > 0) { + i -= 1; + const cb = self.exit_cbs.items[i]; + cb(self); + } + self.exit_cbs.clearRetainingCapacity(); } pub fn idempotencyKey(self: *CtxBase) []const u8 { @@ -304,6 +319,8 @@ pub fn CtxView(comptime spec: anytype) type { return struct { base: *CtxBase, + pub const __reads = reads; + pub const __writes = writes; /// Require a slot to be populated (must be in .reads or .writes) /// Returns error.SlotMissing if the slot was not previously written diff --git a/src/zerver/impure/executor.zig b/src/zerver/impure/executor.zig index 2b88b49..e484468 100644 --- a/src/zerver/impure/executor.zig +++ b/src/zerver/impure/executor.zig @@ -41,6 +41,7 @@ const ReactorNeedRunner = struct { required: bool, token: u32, telemetry_sequence: usize, + queue_label: []const u8, }; allocator: std.mem.Allocator, @@ -64,8 +65,8 @@ const ReactorNeedRunner = struct { insert_error: ?error{OutOfMemory} = null, join_state: ?reactor_join.JoinState = null, join_status: ?reactor_join.Status = null, - continuation_decision: ?types.Decision = null, - completed_continuation_ctx: ?*ContinuationJobContext = null, + step_decision: ?types.Decision = null, + completed_step_ctx: ?*StepJobContext = null, fn run(self: *ReactorNeedRunner) !types.Decision { self.results = std.AutoHashMap(u32, Completion).init(self.allocator); @@ -77,8 +78,8 @@ const ReactorNeedRunner = struct { self.last_failure_error = null; self.insert_error = null; self.join_status = null; - self.continuation_decision = null; - self.completed_continuation_ctx = null; + self.step_decision = null; + self.completed_step_ctx = null; self.join_state = if (self.outstanding > 0) reactor_join.JoinState.init(.{ .mode = self.need.mode, @@ -97,7 +98,7 @@ const ReactorNeedRunner = struct { if (self.outstanding == 0) { if (self.telemetry_ctx) |t| { - t.continuationResume(self.need_sequence, @intFromPtr(self.need.continuation), self.need.mode, self.need.join); + t.stepResume(self.need_sequence, @intFromPtr(self.need.continuation), self.need.mode, self.need.join); } slog.debug("reactor_need_immediate_resume", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), @@ -155,16 +156,16 @@ const ReactorNeedRunner = struct { } if (self.telemetry_ctx) |t| { - t.continuationResume(self.need_sequence, @intFromPtr(self.need.continuation), self.need.mode, self.need.join); + t.stepResume(self.need_sequence, @intFromPtr(self.need.continuation), self.need.mode, self.need.join); } slog.debug("reactor_need_resume_ready", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), - slog.Attr.uint("continuation", @as(u64, @intCast(@intFromPtr(self.need.continuation)))), + slog.Attr.uint("step_ptr", @as(u64, @intCast(@intFromPtr(self.need.continuation)))), }); if (self.task_system) |ts| { - return try self.resumeContinuationViaTaskSystem(ts); + return try self.resumeStepViaTaskSystem(ts); } return self.executor.executeStepInternal(self.ctx_base, self.need.continuation, self.depth + 1); @@ -205,6 +206,7 @@ const ReactorNeedRunner = struct { .required = required, .token = token, .telemetry_sequence = effect_sequence, + .queue_label = self.effector_jobs.label(), }; const job = reactor_jobs.Job{ @@ -224,6 +226,14 @@ const ReactorNeedRunner = struct { if (submit_attempt) |submit_result| { switch (submit_result) { .done => { + job_ctx.queue_label = computeQueueLabel(self); + if (self.telemetry_ctx) |t| { + t.effectJobEnqueued(.{ + .need_sequence = self.need_sequence, + .effect_sequence = effect_sequence, + .queue = job_ctx.queue_label, + }); + } slog.debug("reactor_effect_compute_enqueued", &.{ slog.Attr.string("effect", @tagName(effect_ptr.*)), slog.Attr.uint("token", @as(u64, @intCast(token))), @@ -231,6 +241,7 @@ const ReactorNeedRunner = struct { return; }, .fallback => { + job_ctx.queue_label = self.effector_jobs.label(); slog.debug("reactor_effect_compute_fallback", &.{ slog.Attr.string("effect", @tagName(effect_ptr.*)), slog.Attr.uint("token", @as(u64, @intCast(token))), @@ -257,6 +268,14 @@ const ReactorNeedRunner = struct { slog.Attr.uint("token", @as(u64, @intCast(token))), slog.Attr.string("queue", self.effector_jobs.label()), }); + + if (self.telemetry_ctx) |t| { + t.effectJobEnqueued(.{ + .need_sequence = self.need_sequence, + .effect_sequence = effect_sequence, + .queue = job_ctx.queue_label, + }); + } } fn executeEffect(self: *ReactorNeedRunner, effect_ptr: *const types.Effect, timeout_ms: u32) types.EffectResult { @@ -297,6 +316,8 @@ const ReactorNeedRunner = struct { var failure_details: ?types.Error = null; var is_success = false; + const worker_info = reactor_jobs.currentWorkerInfo(); + switch (result) { .success => |payload| { is_success = true; @@ -320,6 +341,17 @@ const ReactorNeedRunner = struct { slog.Attr.string("error", if (is_success) "" else error_key), }); + if (self.telemetry_ctx) |t| { + t.effectJobCompleted(.{ + .need_sequence = self.need_sequence, + .effect_sequence = job_ctx.telemetry_sequence, + .queue = job_ctx.queue_label, + .success = is_success, + .job_ctx = @intFromPtr(job_ctx), + .worker_index = if (worker_info) |info| info.worker_index else null, + }); + } + self.mutex.lock(); defer self.mutex.unlock(); @@ -418,16 +450,20 @@ const ReactorNeedRunner = struct { return SubmitComputeResult.done; } - const ContinuationJobContext = struct { + const StepJobContext = struct { runner: *ReactorNeedRunner, + queue_label: []const u8, }; - fn resumeContinuationViaTaskSystem(self: *ReactorNeedRunner, ts: *reactor_task_system.TaskSystem) !types.Decision { - const job_ctx = try self.allocator.create(ContinuationJobContext); - job_ctx.* = .{ .runner = self }; + fn resumeStepViaTaskSystem(self: *ReactorNeedRunner, ts: *reactor_task_system.TaskSystem) !types.Decision { + const step_jobs = ts.stepJobs(); + const queue_label = step_jobs.label(); + + const job_ctx = try self.allocator.create(StepJobContext); + job_ctx.* = .{ .runner = self, .queue_label = queue_label }; self.mutex.lock(); - self.continuation_decision = null; + self.step_decision = null; self.mutex.unlock(); slog.debug("reactor_step_context_allocated", &.{ @@ -436,7 +472,7 @@ const ReactorNeedRunner = struct { }); const job = reactor_jobs.Job{ - .callback = continuationJobCallback, + .callback = stepJobCallback, .ctx = @ptrCast(@alignCast(job_ctx)), }; @@ -445,9 +481,9 @@ const ReactorNeedRunner = struct { slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), }); - ts.submitContinuation(job) catch |err| { + ts.submitStep(job) catch |err| { self.allocator.destroy(job_ctx); - const failure = continuationQueueFailure(err); + const failure = stepQueueFailure(err); slog.err("reactor_step_enqueue_failed", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), slog.Attr.string("error", @errorName(err)), @@ -460,22 +496,33 @@ const ReactorNeedRunner = struct { slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), }); - return self.waitForContinuationDecision(); + if (self.telemetry_ctx) |t| { + t.stepJobEnqueued(.{ + .need_sequence = self.need_sequence, + .job_ctx = @intFromPtr(job_ctx), + .queue = queue_label, + }); + } + + return self.waitForStepDecision(); } - fn waitForContinuationDecision(self: *ReactorNeedRunner) types.Decision { + fn waitForStepDecision(self: *ReactorNeedRunner) types.Decision { self.mutex.lock(); - while (self.continuation_decision == null) { + while (self.step_decision == null) { slog.debug("reactor_step_wait", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), }); + if (self.telemetry_ctx) |t| { + t.stepWait(self.need_sequence); + } self.cond.wait(&self.mutex); } - const decision = self.continuation_decision.?; - const job_ctx = self.completed_continuation_ctx; - self.continuation_decision = null; - self.completed_continuation_ctx = null; + const decision = self.step_decision.?; + const job_ctx = self.completed_step_ctx; + self.step_decision = null; + self.completed_step_ctx = null; self.mutex.unlock(); if (job_ctx) |ctx| { @@ -493,9 +540,9 @@ const ReactorNeedRunner = struct { return decision; } - fn finishContinuation(self: *ReactorNeedRunner, decision: types.Decision) void { + fn finishStep(self: *ReactorNeedRunner, decision: types.Decision) void { self.mutex.lock(); - self.continuation_decision = decision; + self.step_decision = decision; self.mutex.unlock(); self.cond.signal(); slog.debug("reactor_step_publish", &.{ @@ -504,14 +551,14 @@ const ReactorNeedRunner = struct { }); } - fn markContinuationJobComplete(self: *ReactorNeedRunner, job_ctx: *ContinuationJobContext) void { + fn markStepJobComplete(self: *ReactorNeedRunner, job_ctx: *StepJobContext) void { self.mutex.lock(); defer self.mutex.unlock(); slog.debug("reactor_step_context_complete", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), }); - self.completed_continuation_ctx = job_ctx; + self.completed_step_ctx = job_ctx; } }; @@ -522,6 +569,16 @@ fn reactorNeedJobCallback(ctx_ptr: *anyopaque) void { slog.Attr.string("effect", @tagName(job_ctx.effect.*)), slog.Attr.uint("token", @as(u64, @intCast(job_ctx.token))), }); + if (runner.telemetry_ctx) |t| { + const worker_info = reactor_jobs.currentWorkerInfo(); + t.effectJobStarted(.{ + .need_sequence = runner.need_sequence, + .effect_sequence = job_ctx.telemetry_sequence, + .queue = job_ctx.queue_label, + .job_ctx = @intFromPtr(job_ctx), + .worker_index = if (worker_info) |info| info.worker_index else null, + }); + } const result = runner.executeEffect(job_ctx.effect, job_ctx.timeout_ms); runner.recordCompletion(job_ctx, result); slog.debug("reactor_effect_job_finish", &.{ @@ -531,8 +588,8 @@ fn reactorNeedJobCallback(ctx_ptr: *anyopaque) void { runner.allocator.destroy(job_ctx); } -fn continuationJobCallback(ctx_ptr: *anyopaque) void { - const job_ctx: *ReactorNeedRunner.ContinuationJobContext = @ptrCast(@alignCast(ctx_ptr)); +fn stepJobCallback(ctx_ptr: *anyopaque) void { + const job_ctx: *ReactorNeedRunner.StepJobContext = @ptrCast(@alignCast(ctx_ptr)); const runner = job_ctx.runner; slog.debug("reactor_step_job_start", &.{ @@ -540,19 +597,50 @@ fn continuationJobCallback(ctx_ptr: *anyopaque) void { slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(job_ctx)))), }); + const worker_info = reactor_jobs.currentWorkerInfo(); + const worker_index_value: ?usize = if (worker_info) |info| info.worker_index else null; + const queue_label = if (worker_info) |info| info.queue else job_ctx.queue_label; + + if (runner.telemetry_ctx) |t| { + t.stepJobStarted(.{ + .need_sequence = runner.need_sequence, + .job_ctx = @intFromPtr(job_ctx), + .queue = queue_label, + .worker_index = worker_index_value, + }); + } + const decision = runner.executor.executeStepInternal(runner.ctx_base, runner.need.continuation, runner.depth + 1) catch |err| { - const failure = failFromCrash(runner.executor, runner.ctx_base, "continuation", err, runner.depth + 1); + const failure = failFromCrash(runner.executor, runner.ctx_base, "step", err, runner.depth + 1); slog.err("reactor_step_job_crash", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(runner.need_sequence))), slog.Attr.string("error", @errorName(err)), }); - runner.markContinuationJobComplete(job_ctx); - runner.finishContinuation(failure); + runner.markStepJobComplete(job_ctx); + if (runner.telemetry_ctx) |t| { + t.stepJobCompleted(.{ + .need_sequence = runner.need_sequence, + .job_ctx = @intFromPtr(job_ctx), + .queue = queue_label, + .worker_index = worker_index_value, + .decision = @tagName(failure), + }); + } + runner.finishStep(failure); return; }; - runner.markContinuationJobComplete(job_ctx); - runner.finishContinuation(decision); + runner.markStepJobComplete(job_ctx); + if (runner.telemetry_ctx) |t| { + t.stepJobCompleted(.{ + .need_sequence = runner.need_sequence, + .job_ctx = @intFromPtr(job_ctx), + .queue = queue_label, + .worker_index = worker_index_value, + .decision = @tagName(decision), + }); + } + runner.finishStep(decision); slog.debug("reactor_step_job_finish", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(runner.need_sequence))), slog.Attr.string("decision", @tagName(decision)), @@ -643,7 +731,7 @@ pub const Executor = struct { 0; decision = self.executeNeed(ctx_base, need, depth + 1, need_sequence) catch |err| { - return failFromCrash(self, ctx_base, "continuation", err, depth + 1); + return failFromCrash(self, ctx_base, "step", err, depth + 1); }; } @@ -682,7 +770,7 @@ pub const Executor = struct { // MVP: execute sequentially regardless of mode // Phase-2 can parallelize this - for (need.effects) |effect| { + effect_loop: for (need.effects) |effect| { const effect_kind = @tagName(effect); const token = effectToken(effect); const timeout_ms = effectTimeout(effect); @@ -703,6 +791,8 @@ pub const Executor = struct { else 0; + var should_break = false; + const result = self.effect_handler(&effect, timeout_ms) catch { const error_result: types.Error = .{ .kind = types.ErrorCode.UpstreamUnavailable, @@ -737,9 +827,13 @@ pub const Executor = struct { .Pending => {}, .Resume => |resume_info| { join_status = resume_info.status; + if (state.isResumed()) { + should_break = true; + } }, } } + if (should_break) break :effect_loop; continue; }; @@ -791,9 +885,14 @@ pub const Executor = struct { .Pending => {}, .Resume => |resume_info| { join_status = resume_info.status; + if (state.isResumed()) { + should_break = true; + } }, } } + + if (should_break) break :effect_loop; } if (total_effects > 0) { @@ -838,7 +937,7 @@ pub const Executor = struct { // Call the continuation function if (self.telemetry_ctx) |t| { - t.continuationResume(need_sequence, @intFromPtr(need.continuation), need.mode, need.join); + t.stepResume(need_sequence, @intFromPtr(need.continuation), need.mode, need.join); } return self.executeStepInternal(ctx_base, need.continuation, depth + 1); @@ -984,6 +1083,14 @@ fn requiresComputePool(effect: types.Effect) bool { }; } +fn computeQueueLabel(self: *ReactorNeedRunner) []const u8 { + const ts = self.task_system orelse return self.effector_jobs.label(); + if (ts.computeJobs()) |compute_jobs| { + return compute_jobs.label(); + } + return ts.stepJobs().label(); +} + fn effectQueueFailure(effect: types.Effect, err: anyerror) types.Error { const kind: u16 = switch (err) { reactor_jobs.SubmitError.QueueFull => types.ErrorCode.TooManyRequests, @@ -995,14 +1102,14 @@ fn effectQueueFailure(effect: types.Effect, err: anyerror) types.Error { }; } -fn continuationQueueFailure(err: anyerror) types.Error { +fn stepQueueFailure(err: anyerror) types.Error { const kind: u16 = switch (err) { reactor_jobs.SubmitError.QueueFull => types.ErrorCode.TooManyRequests, else => types.ErrorCode.UpstreamUnavailable, }; return .{ .kind = kind, - .ctx = .{ .what = "continuation", .key = @errorName(err) }, + .ctx = .{ .what = "step", .key = @errorName(err) }, }; } diff --git a/src/zerver/impure/server.zig b/src/zerver/impure/server.zig index 8c72187..f48cd36 100644 --- a/src/zerver/impure/server.zig +++ b/src/zerver/impure/server.zig @@ -8,12 +8,44 @@ const tracer_module = @import("../observability/tracer.zig"); const slog = @import("../observability/slog.zig"); const http_status = @import("../core/http_status.zig").HttpStatus; const telemetry = @import("../observability/telemetry.zig"); +const net_handler = @import("../runtime/handler.zig"); pub const Address = struct { ip: [4]u8, port: u16, }; +fn shouldKeepConnectionAliveFromRaw(request_data: []const u8) bool { + // Parse Connection header from raw request bytes. Mirrors runtime/listener behaviour. + var lines = std.mem.splitSequence(u8, request_data, "\r\n"); + + // Skip request line + _ = lines.next(); + + while (lines.next()) |line| { + if (line.len == 0) break; + + if (std.ascii.startsWithIgnoreCase(line, "connection:")) { + const value_start = "connection:".len; + if (value_start >= line.len) continue; + + const value = std.mem.trim(u8, line[value_start..], " \t"); + + if (std.ascii.eqlIgnoreCase(value, "close")) { + return false; + } + + if (std.ascii.eqlIgnoreCase(value, "keep-alive")) { + return true; + } + + return true; + } + } + + return true; +} + pub const Config = struct { addr: Address, on_error: *const fn (*ctx_module.CtxBase) anyerror!types.Decision, @@ -142,7 +174,7 @@ pub const Server = struct { pub fn init( allocator: std.mem.Allocator, cfg: Config, - effect_handler: *const fn (*const types.Effect, u32) anyerror!types.EffectResult, + effect_handler: *const fn (*const types.Effect, u32) anyerror!types.EffectResult, ) !Server { return Server{ .allocator = allocator, @@ -742,6 +774,8 @@ pub const Server = struct { .need => types.Response{ .status = http_status.internal_server_error, .body = .{ .complete = "Pipeline incomplete" } }, }; + ctx.runExitCallbacks(); + const response_metrics = telemetry.Telemetry.responseMetricsFromResponse(response); telemetry_ctx.recordResponseMetrics(response_metrics); @@ -808,6 +842,8 @@ pub const Server = struct { else => {}, } + ctx.runExitCallbacks(); + const response_metrics = telemetry.Telemetry.responseMetricsFromResponse(final_response); telemetry_ctx.recordResponseMetrics(response_metrics); @@ -1234,19 +1270,129 @@ pub const Server = struct { /// Start listening for HTTP requests (blocking). pub fn listen(self: *Server) !void { - slog.info("Server starting", &.{ - slog.Attr.uint("ip0", self.config.addr.ip[0]), - slog.Attr.uint("ip1", self.config.addr.ip[1]), - slog.Attr.uint("ip2", self.config.addr.ip[2]), - slog.Attr.uint("ip3", self.config.addr.ip[3]), - slog.Attr.uint("port", self.config.addr.port), + var ip_buf: [32]u8 = undefined; + const ip_str = std.fmt.bufPrint(&ip_buf, "{d}.{d}.{d}.{d}", .{ + self.config.addr.ip[0], + self.config.addr.ip[1], + self.config.addr.ip[2], + self.config.addr.ip[3], + }) catch "0.0.0.0"; + + const listen_addr = std.net.Address.initIp4(self.config.addr.ip, self.config.addr.port); + var listener = try listen_addr.listen(.{ .reuse_address = true }); + defer listener.deinit(); + + slog.info("Server ready for HTTP requests", &.{ + slog.Attr.string("host", ip_str), + slog.Attr.int("port", @as(i64, @intCast(self.config.addr.port))), + slog.Attr.string("status", "running"), }); - // TODO: Phase-2: Implement actual TCP listener - // For MVP, this is a stub that allows testing via handleRequest() - slog.info("MVP server initialized", &.{ - slog.Attr.string("note", "requires explicit handleRequest() calls"), - slog.Attr.string("status", "no TCP listener yet"), - }); + while (true) { + const connection = listener.accept() catch |err| { + slog.err("Failed to accept connection", &.{ + slog.Attr.string("error", @errorName(err)), + }); + continue; + }; + + slog.info("Accepted new connection", &.{}); + + self.handleConnection(connection) catch |err| { + slog.err("Connection handling failed", &.{ + slog.Attr.string("error", @errorName(err)), + }); + }; + } + } + + fn handleConnection(self: *Server, connection: std.net.Server.Connection) !void { + defer connection.stream.close(); + + const keep_alive_timeout_ms: i64 = 60 * 1000; + var last_activity = std.time.milliTimestamp(); + + while (true) { + const now = std.time.milliTimestamp(); + if (now - last_activity > keep_alive_timeout_ms) { + slog.debug("Connection idle timeout", &.{}); + return; + } + + var request_arena = std.heap.ArenaAllocator.init(self.allocator); + defer request_arena.deinit(); + + const request_bytes = net_handler.readRequestWithTimeout( + connection, + request_arena.allocator(), + 5000, + ) catch |err| { + switch (err) { + error.Timeout, error.ConnectionClosed => { + slog.debug("Request read timeout or connection closed", &.{}); + return; + }, + else => { + slog.err("Failed to read request", &.{ + slog.Attr.string("error", @errorName(err)), + }); + return; + }, + } + }; + + if (request_bytes.len == 0) { + slog.debug("Received empty request", &.{}); + return; + } + + last_activity = std.time.milliTimestamp(); + + const preview_len = @min(request_bytes.len, 120); + slog.info("Received HTTP request", &.{ + slog.Attr.uint("bytes", request_bytes.len), + slog.Attr.string("preview", request_bytes[0..preview_len]), + }); + + if (request_bytes.len > 0) { + const line_end = std.mem.indexOf(u8, request_bytes, "\r\n") orelse request_bytes.len; + const request_line = request_bytes[0..line_end]; + slog.info("HTTP request line", &.{ + slog.Attr.string("line", request_line), + }); + } + + const response_result = self.handleRequest(request_bytes, request_arena.allocator()) catch |err| { + slog.err("Failed to handle request", &.{ + slog.Attr.string("error", @errorName(err)), + }); + try net_handler.sendErrorResponse(connection, "500 Internal Server Error", "Internal Server Error"); + return; + }; + + slog.info("handleRequest completed", &.{ + slog.Attr.enumeration("result", response_result), + }); + + switch (response_result) { + .complete => |response| { + try net_handler.sendResponse(connection, response); + slog.info("Response sent successfully", &.{}); + }, + .streaming => |streaming_resp| { + try net_handler.sendStreamingResponse(connection, streaming_resp.headers, streaming_resp.writer, streaming_resp.context); + slog.info("Streaming response initiated", &.{}); + return; + }, + } + + const keep_alive = shouldKeepConnectionAliveFromRaw(request_bytes); + if (!keep_alive) { + slog.info("Connection close requested by client", &.{}); + return; + } + + slog.info("Keeping connection alive for next request", &.{}); + } } }; diff --git a/src/zerver/observability/otel.zig b/src/zerver/observability/otel.zig index 17b861e..531f389 100644 --- a/src/zerver/observability/otel.zig +++ b/src/zerver/observability/otel.zig @@ -252,6 +252,8 @@ const RequestRecord = struct { child_spans: std.ArrayList(*ChildSpan), step_spans: std.AutoHashMap(usize, *ChildSpan), effect_spans: std.AutoHashMap(usize, *ChildSpan), + job_spans: std.AutoHashMap(usize, *ChildSpan), + step_job_spans: std.AutoHashMap(usize, *ChildSpan), step_stack: std.ArrayList(*ChildSpan), fn create(allocator: std.mem.Allocator, event: telemetry.RequestStartEvent) !*RequestRecord { @@ -278,6 +280,8 @@ const RequestRecord = struct { .child_spans = try std.ArrayList(*ChildSpan).initCapacity(allocator, 4), .step_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), .effect_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), + .job_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), + .step_job_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), .step_stack = try std.ArrayList(*ChildSpan).initCapacity(allocator, 4), }; errdefer { @@ -334,6 +338,8 @@ const RequestRecord = struct { evt.deinit(); } self.events.deinit(self.allocator); + self.job_spans.deinit(); + self.step_job_spans.deinit(); self.* = undefined; } @@ -371,8 +377,8 @@ const RequestRecord = struct { try self.finishEffectSpan(event); } - fn recordContinuation(self: *RequestRecord, event: telemetry.ContinuationEvent) !void { - var req_event = try RequestEvent.init(self.allocator, "zerver.continuation_resume", nowUnixNano()); + fn recordStepResume(self: *RequestRecord, event: telemetry.StepResumeEvent) !void { + var req_event = try RequestEvent.init(self.allocator, "zerver.step_resume", nowUnixNano()); try req_event.addAttribute(try Attribute.initInt(self.allocator, "need.sequence", @as(i64, @intCast(event.need_sequence)))); try req_event.addAttribute(try Attribute.initInt(self.allocator, "resume.ptr", @as(i64, @intCast(event.resume_ptr)))); try req_event.addAttribute(try Attribute.initString(self.allocator, "need.mode", @tagName(event.mode))); @@ -503,6 +509,304 @@ const RequestRecord = struct { } } + fn recordEffectJobEnqueued(self: *RequestRecord, event: telemetry.EffectJobEnqueuedEvent) !void { + const effect_parent = self.effect_spans.get(event.effect_sequence); + const job_span = try self.ensureJobSpan(effect_parent, event.need_sequence, event.effect_sequence, event.queue, event.timestamp_ms); + + var job_event = try self.buildJobStageEvent( + "zerver.effect_job_enqueued", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "enqueued", + null, + ); + var committed = false; + defer if (!committed) job_event.deinit(); + + try job_span.pushEvent(job_event); + committed = true; + + if (effect_parent) |parent_span| { + var mirror_event = try self.buildJobStageEvent( + "zerver.effect_job_enqueued", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "enqueued", + null, + ); + var mirror_committed = false; + defer if (!mirror_committed) mirror_event.deinit(); + try parent_span.pushEvent(mirror_event); + mirror_committed = true; + } + } + + fn recordEffectJobStarted(self: *RequestRecord, event: telemetry.EffectJobStartedEvent) !void { + const effect_parent = self.effect_spans.get(event.effect_sequence); + const job_span = try self.ensureJobSpan(effect_parent, event.need_sequence, event.effect_sequence, event.queue, event.timestamp_ms); + + var job_event = try self.buildJobStageEvent( + "zerver.effect_job_started", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "started", + null, + ); + var committed = false; + defer if (!committed) job_event.deinit(); + + if (event.job_ctx) |ctx| { + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + } + if (event.worker_index) |worker| { + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + + try job_span.pushEvent(job_event); + committed = true; + + if (effect_parent) |parent_span| { + var mirror_event = try self.buildJobStageEvent( + "zerver.effect_job_started", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "started", + null, + ); + var mirror_committed = false; + defer if (!mirror_committed) mirror_event.deinit(); + if (event.job_ctx) |ctx| { + try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + } + if (event.worker_index) |worker| { + try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + try parent_span.pushEvent(mirror_event); + mirror_committed = true; + } + } + + fn recordEffectJobCompleted(self: *RequestRecord, event: telemetry.EffectJobCompletedEvent) !void { + const effect_parent = self.effect_spans.get(event.effect_sequence); + const job_span = try self.ensureJobSpan(effect_parent, event.need_sequence, event.effect_sequence, event.queue, event.timestamp_ms); + + job_span.end_time_unix_ns = event.timestamp_ms * std.time.ns_per_ms; + try job_span.pushAttribute(try Attribute.initBool(self.allocator, "job.success", event.success)); + var job_event = try self.buildJobStageEvent( + "zerver.effect_job_completed", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "completed", + event.success, + ); + var committed = false; + defer if (!committed) job_event.deinit(); + if (event.job_ctx) |ctx| { + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + } + if (event.worker_index) |worker| { + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + try job_span.pushEvent(job_event); + committed = true; + + if (effect_parent) |parent_span| { + var mirror_event = try self.buildJobStageEvent( + "zerver.effect_job_completed", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "completed", + event.success, + ); + var mirror_committed = false; + defer if (!mirror_committed) mirror_event.deinit(); + if (event.job_ctx) |ctx| { + try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + } + if (event.worker_index) |worker| { + try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + try parent_span.pushEvent(mirror_event); + mirror_committed = true; + } + + if (event.success) { + if (job_span.status != .@"error") { + job_span.status = .ok; + } + } else { + job_span.setStatus(.@"error", "job failed") catch {}; + } + + const total_ns = if (job_span.end_time_unix_ns > job_span.start_time_unix_ns) + job_span.end_time_unix_ns - job_span.start_time_unix_ns + else + 0; + const total_ms = @divTrunc(total_ns, std.time.ns_per_ms); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.total_duration_ms", @as(i64, @intCast(total_ms)))); + + _ = self.job_spans.remove(event.effect_sequence); + } + + fn recordStepJobEnqueued(self: *RequestRecord, event: telemetry.StepJobEnqueuedEvent) !void { + const job_span = try self.ensureStepJobSpan(event.need_sequence, event.job_ctx, event.queue, event.timestamp_ms); + + var job_event = try self.buildStepJobStageEvent( + "zerver.step_job_enqueued", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "enqueued", + null, + null, + ); + var committed = false; + defer if (!committed) job_event.deinit(); + try job_span.pushEvent(job_event); + committed = true; + + var request_event = try self.buildStepJobStageEvent( + "zerver.step_job_enqueued", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "enqueued", + null, + null, + ); + var request_committed = false; + defer if (!request_committed) request_event.deinit(); + try self.pushEvent(request_event); + request_committed = true; + } + + fn recordStepJobStarted(self: *RequestRecord, event: telemetry.StepJobStartedEvent) !void { + const job_span = try self.ensureStepJobSpan(event.need_sequence, event.job_ctx, event.queue, event.timestamp_ms); + + if (event.worker_index) |worker| { + var has_worker_attr = false; + for (job_span.attributes.items) |attr| { + if (std.mem.eql(u8, attr.key, "job.worker_index")) { + has_worker_attr = true; + break; + } + } + if (!has_worker_attr) { + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + } + + var job_event = try self.buildStepJobStageEvent( + "zerver.step_job_started", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "started", + event.worker_index, + null, + ); + var committed = false; + defer if (!committed) job_event.deinit(); + try job_span.pushEvent(job_event); + committed = true; + + var request_event = try self.buildStepJobStageEvent( + "zerver.step_job_started", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "started", + event.worker_index, + null, + ); + var request_committed = false; + defer if (!request_committed) request_event.deinit(); + try self.pushEvent(request_event); + request_committed = true; + } + + fn recordStepJobCompleted(self: *RequestRecord, event: telemetry.StepJobCompletedEvent) !void { + const job_span = try self.ensureStepJobSpan(event.need_sequence, event.job_ctx, event.queue, event.timestamp_ms); + + if (event.worker_index) |worker| { + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + try job_span.pushAttribute(try Attribute.initString(self.allocator, "job.decision", event.decision)); + + job_span.end_time_unix_ns = event.timestamp_ms * std.time.ns_per_ms; + if (job_span.status != .@"error") { + job_span.status = .ok; + } + + var job_event = try self.buildStepJobStageEvent( + "zerver.step_job_completed", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "completed", + event.worker_index, + event.decision, + ); + var committed = false; + defer if (!committed) job_event.deinit(); + try job_span.pushEvent(job_event); + committed = true; + + var request_event = try self.buildStepJobStageEvent( + "zerver.step_job_completed", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "completed", + event.worker_index, + event.decision, + ); + var request_committed = false; + defer if (!request_committed) request_event.deinit(); + try self.pushEvent(request_event); + request_committed = true; + + const total_ns = if (job_span.end_time_unix_ns > job_span.start_time_unix_ns) + job_span.end_time_unix_ns - job_span.start_time_unix_ns + else + 0; + const total_ms = @divTrunc(total_ns, std.time.ns_per_ms); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.total_duration_ms", @as(i64, @intCast(total_ms)))); + + _ = self.step_job_spans.remove(event.job_ctx); + } + + fn recordStepWait(self: *RequestRecord, event: telemetry.StepWaitEvent) !void { + var req_event = try RequestEvent.init(self.allocator, "zerver.step_wait", event.timestamp_ms * std.time.ns_per_ms); + var committed = false; + defer if (!committed) req_event.deinit(); + try req_event.addAttribute(try Attribute.initString(self.allocator, "job.type", "step")); + try req_event.addAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(event.need_sequence)))); + try req_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "waiting")); + try self.pushEvent(req_event); + committed = true; + } + fn removeActiveStep(self: *RequestRecord, span: *ChildSpan) void { var i: usize = self.step_stack.items.len; while (i > 0) { @@ -537,9 +841,33 @@ const RequestRecord = struct { } } + var job_iter = self.job_spans.iterator(); + while (job_iter.next()) |entry| { + const span = entry.value_ptr.*; + if (span.end_time_unix_ns <= span.start_time_unix_ns) { + span.end_time_unix_ns = end_time_unix_ns; + } + if (span.status != .@"error") { + span.setStatus(.@"error", "job span incomplete") catch {}; + } + } + + var step_job_iter = self.step_job_spans.iterator(); + while (step_job_iter.next()) |entry| { + const span = entry.value_ptr.*; + if (span.end_time_unix_ns <= span.start_time_unix_ns) { + span.end_time_unix_ns = end_time_unix_ns; + } + if (span.status != .@"error") { + span.setStatus(.@"error", "step job span incomplete") catch {}; + } + } + self.step_stack.clearRetainingCapacity(); self.step_spans.clearRetainingCapacity(); self.effect_spans.clearRetainingCapacity(); + self.job_spans.clearRetainingCapacity(); + self.step_job_spans.clearRetainingCapacity(); } fn applyRequestEnd(self: *RequestRecord, event: telemetry.RequestEndEvent) !void { @@ -583,6 +911,153 @@ const RequestRecord = struct { } self.status_message = try self.allocator.dupe(u8, message); } + + fn ensureJobSpan( + self: *RequestRecord, + effect_parent: ?*ChildSpan, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + timestamp_ms: u64, + ) !*ChildSpan { + if (self.job_spans.get(effect_sequence)) |span| { + const timestamp_ns = timestamp_ms * std.time.ns_per_ms; + if (timestamp_ns < span.start_time_unix_ns) { + span.start_time_unix_ns = timestamp_ns; + } + return span; + } + + const parent_id: [8]u8 = if (effect_parent) |parent| + parent.span_id + else + self.span_id; + const start_ns = timestamp_ms * std.time.ns_per_ms; + + var span = try ChildSpan.create(self.allocator, "effect_job", .internal, parent_id, start_ns); + errdefer { + span.deinit(); + self.allocator.destroy(span); + } + + try span.pushAttribute(try Attribute.initString(self.allocator, "job.type", "effect")); + try span.pushAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); + try span.pushAttribute(try Attribute.initInt(self.allocator, "job.effect_sequence", @as(i64, @intCast(effect_sequence)))); + try span.pushAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); + + self.child_spans.append(self.allocator, span) catch |err| { + span.deinit(); + self.allocator.destroy(span); + return err; + }; + + self.job_spans.put(effect_sequence, span) catch |err| { + _ = self.child_spans.pop(); + span.deinit(); + self.allocator.destroy(span); + return err; + }; + + return span; + } + + fn ensureStepJobSpan( + self: *RequestRecord, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + timestamp_ms: u64, + ) !*ChildSpan { + if (self.step_job_spans.get(job_ctx)) |span| { + const timestamp_ns = timestamp_ms * std.time.ns_per_ms; + if (timestamp_ns < span.start_time_unix_ns) { + span.start_time_unix_ns = timestamp_ns; + } + return span; + } + + const parent_id: [8]u8 = if (self.step_stack.items.len != 0) + self.step_stack.items[self.step_stack.items.len - 1].span_id + else + self.span_id; + const start_ns = timestamp_ms * std.time.ns_per_ms; + + var span = try ChildSpan.create(self.allocator, "step_job", .internal, parent_id, start_ns); + errdefer { + span.deinit(); + self.allocator.destroy(span); + } + + try span.pushAttribute(try Attribute.initString(self.allocator, "job.type", "step")); + try span.pushAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); + try span.pushAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(job_ctx)))); + try span.pushAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); + + self.child_spans.append(self.allocator, span) catch |err| { + span.deinit(); + self.allocator.destroy(span); + return err; + }; + + self.step_job_spans.put(job_ctx, span) catch |err| { + _ = self.child_spans.pop(); + span.deinit(); + self.allocator.destroy(span); + return err; + }; + + return span; + } + + fn buildJobStageEvent( + self: *RequestRecord, + name: []const u8, + timestamp_ms: u64, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + stage: []const u8, + success: ?bool, + ) !RequestEvent { + var event = try RequestEvent.init(self.allocator, name, timestamp_ms * std.time.ns_per_ms); + errdefer event.deinit(); + try event.addAttribute(try Attribute.initString(self.allocator, "job.type", "effect")); + try event.addAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); + try event.addAttribute(try Attribute.initInt(self.allocator, "job.effect_sequence", @as(i64, @intCast(effect_sequence)))); + try event.addAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); + try event.addAttribute(try Attribute.initString(self.allocator, "job.stage", stage)); + if (success) |value| { + try event.addAttribute(try Attribute.initBool(self.allocator, "job.success", value)); + } + return event; + } + + fn buildStepJobStageEvent( + self: *RequestRecord, + name: []const u8, + timestamp_ms: u64, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + stage: []const u8, + worker_index: ?usize, + decision: ?[]const u8, + ) !RequestEvent { + var event = try RequestEvent.init(self.allocator, name, timestamp_ms * std.time.ns_per_ms); + errdefer event.deinit(); + try event.addAttribute(try Attribute.initString(self.allocator, "job.type", "step")); + try event.addAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); + try event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(job_ctx)))); + try event.addAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); + try event.addAttribute(try Attribute.initString(self.allocator, "job.stage", stage)); + if (worker_index) |worker| { + try event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + if (decision) |value| { + try event.addAttribute(try Attribute.initString(self.allocator, "job.decision", value)); + } + return event; + } }; /// OTLP exporter subscribing to telemetry events and pushing JSON over HTTP. @@ -758,9 +1233,9 @@ pub const OtelExporter = struct { try record.recordEffectEnd(payload); } }, - .continuation_resume => |payload| { + .step_resume => |payload| { if (self.requests.get(payload.request_id)) |record| { - try record.recordContinuation(payload); + try record.recordStepResume(payload); } }, .executor_crash => |payload| { @@ -768,6 +1243,41 @@ pub const OtelExporter = struct { try record.recordExecutorCrash(payload); } }, + .effect_job_enqueued => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordEffectJobEnqueued(payload); + } + }, + .effect_job_started => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordEffectJobStarted(payload); + } + }, + .effect_job_completed => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordEffectJobCompleted(payload); + } + }, + .step_job_enqueued => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordStepJobEnqueued(payload); + } + }, + .step_job_started => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordStepJobStarted(payload); + } + }, + .step_job_completed => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordStepJobCompleted(payload); + } + }, + .step_wait => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordStepWait(payload); + } + }, } if (to_export) |record| { diff --git a/src/zerver/observability/telemetry.zig b/src/zerver/observability/telemetry.zig index 174d290..d359a1b 100644 --- a/src/zerver/observability/telemetry.zig +++ b/src/zerver/observability/telemetry.zig @@ -115,8 +115,8 @@ pub const EffectEndEvent = struct { error_ctx: ?types.ErrorCtx, }; -/// Fired when the executor resumes a continuation after effects complete. -pub const ContinuationEvent = struct { +/// Fired when the executor resumes a paused step after effects complete. +pub const StepResumeEvent = struct { request_id: []const u8, need_sequence: usize, resume_ptr: usize, @@ -124,13 +124,82 @@ pub const ContinuationEvent = struct { join: types.Join, }; -/// Fired when the executor encounters an unexpected crash while running a step or continuation. +/// Fired when the executor encounters an unexpected crash while running a step. pub const ExecutorCrashEvent = struct { request_id: []const u8, phase: []const u8, error_name: []const u8, }; +/// Fired when an effect job is enqueued onto a job system queue. +pub const EffectJobEnqueuedEvent = struct { + request_id: []const u8, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + timestamp_ms: u64, +}; + +/// Fired when a job system worker begins executing an effect job. +pub const EffectJobStartedEvent = struct { + request_id: []const u8, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + job_ctx: ?usize, + worker_index: ?usize, + timestamp_ms: u64, +}; + +/// Fired when a job system worker completes execution of an effect job. +pub const EffectJobCompletedEvent = struct { + request_id: []const u8, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + success: bool, + job_ctx: ?usize, + worker_index: ?usize, + timestamp_ms: u64, +}; + +/// Fired when a step is enqueued for asynchronous execution. +pub const StepJobEnqueuedEvent = struct { + request_id: []const u8, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + timestamp_ms: u64, +}; + +/// Fired when a worker dequeues and begins executing a step job. +pub const StepJobStartedEvent = struct { + request_id: []const u8, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: ?usize, + timestamp_ms: u64, +}; + +/// Fired when a step job completes and yields a decision. +pub const StepJobCompletedEvent = struct { + request_id: []const u8, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: ?usize, + decision: []const u8, + timestamp_ms: u64, +}; + +/// Fired when the main thread parks waiting for a step job result. +pub const StepWaitEvent = struct { + request_id: []const u8, + need_sequence: usize, + timestamp_ms: u64, +}; + /// Union of all telemetry signals publishable to subscribers. pub const Event = union(enum) { request_start: RequestStartEvent, @@ -140,8 +209,15 @@ pub const Event = union(enum) { need_scheduled: NeedScheduledEvent, effect_start: EffectStartEvent, effect_end: EffectEndEvent, - continuation_resume: ContinuationEvent, + step_resume: StepResumeEvent, executor_crash: ExecutorCrashEvent, + effect_job_enqueued: EffectJobEnqueuedEvent, + effect_job_started: EffectJobStartedEvent, + effect_job_completed: EffectJobCompletedEvent, + step_job_enqueued: StepJobEnqueuedEvent, + step_job_started: StepJobStartedEvent, + step_job_completed: StepJobCompletedEvent, + step_wait: StepWaitEvent, }; /// Options supplied when building per-request telemetry. @@ -538,21 +614,224 @@ pub const Telemetry = struct { } }); } - pub fn continuationResume(self: *Telemetry, need_sequence: usize, resume_ptr: usize, mode: types.Mode, join: types.Join) void { - self.logDebug("continuation_resume", &.{ + pub const EffectJobEnqueuedDetails = struct { + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + }; + + pub fn effectJobEnqueued(self: *Telemetry, details: EffectJobEnqueuedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("effect_job_enqueued", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("effect_sequence", details.effect_sequence), + slog.Attr.string("queue", details.queue), + }); + + self.emit(.{ .effect_job_enqueued = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .effect_sequence = details.effect_sequence, + .queue = details.queue, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + + self.tracer.recordEffectJobQueued(details.need_sequence, details.effect_sequence, details.queue); + } + + pub const EffectJobStartedDetails = struct { + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + job_ctx: ?usize = null, + worker_index: ?usize = null, + }; + + pub fn effectJobStarted(self: *Telemetry, details: EffectJobStartedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("effect_job_started", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("effect_sequence", details.effect_sequence), + slog.Attr.string("queue", details.queue), + }); + + self.emit(.{ .effect_job_started = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .effect_sequence = details.effect_sequence, + .queue = details.queue, + .job_ctx = details.job_ctx, + .worker_index = details.worker_index, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + + self.tracer.recordEffectJobStarted( + details.need_sequence, + details.effect_sequence, + details.queue, + details.job_ctx, + details.worker_index, + ); + } + + pub const EffectJobCompletedDetails = struct { + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + success: bool, + job_ctx: ?usize = null, + worker_index: ?usize = null, + }; + + pub fn effectJobCompleted(self: *Telemetry, details: EffectJobCompletedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("effect_job_completed", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("effect_sequence", details.effect_sequence), + slog.Attr.string("queue", details.queue), + slog.Attr.bool("success", details.success), + }); + + self.emit(.{ .effect_job_completed = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .effect_sequence = details.effect_sequence, + .queue = details.queue, + .success = details.success, + .job_ctx = details.job_ctx, + .worker_index = details.worker_index, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + + self.tracer.recordEffectJobCompleted( + details.need_sequence, + details.effect_sequence, + details.queue, + details.success, + details.job_ctx, + details.worker_index, + ); + } + + pub const StepJobEnqueuedDetails = struct { + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + }; + + pub fn stepJobEnqueued(self: *Telemetry, details: StepJobEnqueuedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("step_job_enqueued", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("job_ctx", @as(u64, @intCast(details.job_ctx))), + slog.Attr.string("queue", details.queue), + }); + + self.emit(.{ .step_job_enqueued = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .job_ctx = details.job_ctx, + .queue = details.queue, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + + self.tracer.recordStepJobEnqueued(details.need_sequence, details.job_ctx, details.queue); + } + + pub const StepJobStartedDetails = struct { + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: ?usize = null, + }; + + pub fn stepJobStarted(self: *Telemetry, details: StepJobStartedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("step_job_started", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("job_ctx", @as(u64, @intCast(details.job_ctx))), + slog.Attr.string("queue", details.queue), + }); + + self.emit(.{ .step_job_started = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .job_ctx = details.job_ctx, + .queue = details.queue, + .worker_index = details.worker_index, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + + self.tracer.recordStepJobStarted(details.need_sequence, details.job_ctx, details.queue, details.worker_index); + } + + pub const StepJobCompletedDetails = struct { + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: ?usize = null, + decision: []const u8, + }; + + pub fn stepJobCompleted(self: *Telemetry, details: StepJobCompletedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("step_job_completed", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("job_ctx", @as(u64, @intCast(details.job_ctx))), + slog.Attr.string("queue", details.queue), + slog.Attr.string("decision", details.decision), + }); + + self.emit(.{ .step_job_completed = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .job_ctx = details.job_ctx, + .queue = details.queue, + .worker_index = details.worker_index, + .decision = details.decision, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + + self.tracer.recordStepJobCompleted(details.need_sequence, details.job_ctx, details.queue, details.worker_index, details.decision); + } + + pub fn stepWait(self: *Telemetry, need_sequence: usize) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("step_wait", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", need_sequence), + }); + + self.emit(.{ .step_wait = .{ + .request_id = self.request_id, + .need_sequence = need_sequence, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + + self.tracer.recordStepWait(need_sequence); + } + + pub fn stepResume(self: *Telemetry, need_sequence: usize, resume_ptr: usize, mode: types.Mode, join: types.Join) void { + self.logDebug("step_resume", &.{ slog.Attr.string("request_id", self.request_id), slog.Attr.uint("need_sequence", need_sequence), slog.Attr.uint("resume_ptr", resume_ptr), slog.Attr.string("mode", @tagName(mode)), slog.Attr.string("join", @tagName(join)), }); - self.tracer.recordContinuationResume( + self.tracer.recordStepResume( need_sequence, resume_ptr, @tagName(mode), @tagName(join), ); - self.emit(.{ .continuation_resume = .{ + self.emit(.{ .step_resume = .{ .request_id = self.request_id, .need_sequence = need_sequence, .resume_ptr = resume_ptr, diff --git a/src/zerver/observability/tracer.zig b/src/zerver/observability/tracer.zig index 4ab2599..8e7419f 100644 --- a/src/zerver/observability/tracer.zig +++ b/src/zerver/observability/tracer.zig @@ -19,8 +19,15 @@ pub const EventKind = enum { effect_start, effect_end, need_scheduled, - continuation_resume, + step_resume, request_end, + effect_job_enqueued, + effect_job_started, + effect_job_completed, + step_job_enqueued, + step_job_started, + step_job_completed, + step_wait, }; /// A single trace event. @@ -36,6 +43,12 @@ pub const TraceEvent = struct { resume_ptr: ?usize = null, mode: ?[]const u8 = null, join: ?[]const u8 = null, + effect_sequence: ?usize = null, + job_queue: ?[]const u8 = null, + job_success: ?bool = null, + job_ctx: ?usize = null, + job_worker_index: ?usize = null, + job_decision: ?[]const u8 = null, // TODO: Memory/Safety - Ensure that string slices stored in TraceEvent (step_name, effect_kind, status, error_msg) have a lifetime at least as long as the Tracer itself, or are duplicated into the Tracer's allocator to prevent use-after-free issues. }; @@ -133,7 +146,7 @@ pub const Tracer = struct { } /// Record continuation resume event. - pub fn recordContinuationResume( + pub fn recordStepResume( self: *Tracer, need_sequence: usize, resume_ptr: usize, @@ -141,7 +154,7 @@ pub const Tracer = struct { join: []const u8, ) void { self.recordEvent(.{ - .kind = .continuation_resume, + .kind = .step_resume, .need_sequence = need_sequence, .resume_ptr = resume_ptr, .mode = mode, @@ -160,6 +173,111 @@ pub const Tracer = struct { }); } + pub fn recordEffectJobQueued(self: *Tracer, need_sequence: usize, effect_sequence: usize, queue: []const u8) void { + self.recordEvent(.{ + .kind = .effect_job_enqueued, + .timestamp_ms = 0, + .need_sequence = need_sequence, + .effect_sequence = effect_sequence, + .job_queue = queue, + }); + } + + pub fn recordEffectJobStarted( + self: *Tracer, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + job_ctx: ?usize, + worker_index: ?usize, + ) void { + self.recordEvent(.{ + .kind = .effect_job_started, + .timestamp_ms = 0, + .need_sequence = need_sequence, + .effect_sequence = effect_sequence, + .job_queue = queue, + .job_ctx = job_ctx, + .job_worker_index = worker_index, + }); + } + + pub fn recordEffectJobCompleted( + self: *Tracer, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + success: bool, + job_ctx: ?usize, + worker_index: ?usize, + ) void { + self.recordEvent(.{ + .kind = .effect_job_completed, + .timestamp_ms = 0, + .need_sequence = need_sequence, + .effect_sequence = effect_sequence, + .job_queue = queue, + .job_success = success, + .job_ctx = job_ctx, + .job_worker_index = worker_index, + .status = if (success) "success" else "failure", + }); + } + + pub fn recordStepJobEnqueued(self: *Tracer, need_sequence: usize, job_ctx: usize, queue: []const u8) void { + self.recordEvent(.{ + .kind = .step_job_enqueued, + .timestamp_ms = 0, + .need_sequence = need_sequence, + .job_ctx = job_ctx, + .job_queue = queue, + }); + } + + pub fn recordStepJobStarted( + self: *Tracer, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: ?usize, + ) void { + self.recordEvent(.{ + .kind = .step_job_started, + .timestamp_ms = 0, + .need_sequence = need_sequence, + .job_ctx = job_ctx, + .job_queue = queue, + .job_worker_index = worker_index, + }); + } + + pub fn recordStepJobCompleted( + self: *Tracer, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: ?usize, + decision: []const u8, + ) void { + self.recordEvent(.{ + .kind = .step_job_completed, + .timestamp_ms = 0, + .need_sequence = need_sequence, + .job_ctx = job_ctx, + .job_queue = queue, + .job_worker_index = worker_index, + .job_decision = decision, + }); + } + + pub fn recordStepWait(self: *Tracer, need_sequence: usize) void { + self.recordEvent(.{ + .kind = .step_wait, + .timestamp_ms = 0, + .need_sequence = need_sequence, + }); + } + /// Record an error. pub fn recordError(self: *Tracer, msg: []const u8) void { // TODO: Logical Error - The 'recordError' function currently records an event of kind '.request_end'. This might overwrite a legitimate request_end event or cause confusion in the trace. Consider a distinct event type for errors or adding error details to an existing event. @@ -253,6 +371,42 @@ pub const Tracer = struct { try writeJsonString(&writer, join); } + if (event.effect_sequence) |effect_sequence| { + try writer.writeByte(','); + try writer.writeAll("\"effect_sequence\":"); + try writer.print("{}", .{effect_sequence}); + } + + if (event.job_queue) |queue| { + try writer.writeByte(','); + try writer.writeAll("\"job_queue\":"); + try writeJsonString(&writer, queue); + } + + if (event.job_success) |success| { + try writer.writeByte(','); + try writer.writeAll("\"job_success\":"); + try writer.writeAll(if (success) "true" else "false"); + } + + if (event.job_ctx) |job_context| { + try writer.writeByte(','); + try writer.writeAll("\"job_ctx\":"); + try writer.print("{}", .{job_context}); + } + + if (event.job_worker_index) |worker_idx| { + try writer.writeByte(','); + try writer.writeAll("\"job_worker_index\":"); + try writer.print("{}", .{worker_idx}); + } + + if (event.job_decision) |job_decision| { + try writer.writeByte(','); + try writer.writeAll("\"job_decision\":"); + try writeJsonString(&writer, job_decision); + } + try writer.writeByte('}'); } diff --git a/src/zerver/runtime/handler.zig b/src/zerver/runtime/handler.zig index 114124e..540f96e 100644 --- a/src/zerver/runtime/handler.zig +++ b/src/zerver/runtime/handler.zig @@ -5,7 +5,6 @@ const std = @import("std"); const builtin = @import("builtin"); const windows_sockets = @import("platform/windows_sockets.zig"); -const root = @import("../root.zig"); const slog = @import("../observability/slog.zig"); fn hexPreview(data: []const u8, out: []u8) []const u8 { diff --git a/src/zerver/runtime/reactor/job_system.zig b/src/zerver/runtime/reactor/job_system.zig index 83ca978..371d3c1 100644 --- a/src/zerver/runtime/reactor/job_system.zig +++ b/src/zerver/runtime/reactor/job_system.zig @@ -16,6 +16,11 @@ pub const Job = struct { ctx: *anyopaque, }; +pub const WorkerInfo = struct { + queue: []const u8, + worker_index: usize, +}; + pub const InitOptions = struct { allocator: std.mem.Allocator, worker_count: usize, @@ -136,6 +141,10 @@ pub const JobSystem = struct { } fn workerMain(self: *JobSystem, worker_index: usize) !void { + const prev_state = tls_worker_state; + tls_worker_state = .{ .system = self, .worker_index = worker_index }; + defer tls_worker_state = prev_state; + slog.debug("job_worker_start", &.{ slog.Attr.string("queue", self.queue_label), slog.Attr.uint("worker", @as(u64, @intCast(worker_index))), @@ -216,6 +225,23 @@ pub const JobSystem = struct { } }; +const WorkerState = struct { + system: *JobSystem, + worker_index: usize, +}; + +threadlocal var tls_worker_state: ?WorkerState = null; + +pub fn currentWorkerInfo() ?WorkerInfo { + if (tls_worker_state) |state| { + return WorkerInfo{ + .queue = state.system.label(), + .worker_index = state.worker_index, + }; + } + return null; +} + const JobQueue = struct { allocator: std.mem.Allocator, buffer: []Job, diff --git a/src/zerver/runtime/reactor/task_system.zig b/src/zerver/runtime/reactor/task_system.zig index d5f06b3..dc0b060 100644 --- a/src/zerver/runtime/reactor/task_system.zig +++ b/src/zerver/runtime/reactor/task_system.zig @@ -33,7 +33,7 @@ pub const TaskSystem = struct { .allocator = config.allocator, .worker_count = config.continuation_workers, .queue_capacity = config.continuation_queue_capacity, - .label = "continuation_jobs", + .label = "step_jobs", }); errdefer self.continuation.deinit(); @@ -57,7 +57,7 @@ pub const TaskSystem = struct { } slog.debug("task_system_init", &.{ - slog.Attr.string("continuation_queue", self.continuation.label()), + slog.Attr.string("step_queue", self.continuation.label()), slog.Attr.string("compute_kind", @tagName(self.compute_kind)), slog.Attr.bool("has_compute", self.has_compute), }); @@ -77,14 +77,14 @@ pub const TaskSystem = struct { self.continuation.shutdown(); } - pub fn submitContinuation(self: *TaskSystem, task: job.Job) TaskSystemError!void { - slog.debug("task_submit_continuation", &.{ + pub fn submitStep(self: *TaskSystem, task: job.Job) TaskSystemError!void { + slog.debug("task_submit_step", &.{ slog.Attr.string("queue", self.continuation.label()), slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(task.ctx)))), slog.Attr.uint("job_cb", @as(u64, @intCast(@intFromPtr(task.callback)))), }); self.continuation.submit(task) catch |err| { - slog.err("task_submit_continuation_failed", &.{ + slog.err("task_submit_step_failed", &.{ slog.Attr.string("queue", self.continuation.label()), slog.Attr.string("error", @errorName(err)), slog.Attr.uint("job_ctx", @as(u64, @intCast(@intFromPtr(task.ctx)))), @@ -127,7 +127,7 @@ pub const TaskSystem = struct { }; } - pub fn continuationJobs(self: *TaskSystem) *job.JobSystem { + pub fn stepJobs(self: *TaskSystem) *job.JobSystem { return &self.continuation; } diff --git a/src/zerver/runtime/scheduler.zig b/src/zerver/runtime/scheduler.zig index fc78fda..7995ee6 100644 --- a/src/zerver/runtime/scheduler.zig +++ b/src/zerver/runtime/scheduler.zig @@ -54,16 +54,16 @@ pub const Scheduler = struct { self.initialized = false; } - pub fn submitContinuation(self: *Scheduler, job: job_system.Job) task_system.TaskSystemError!void { - return self.inner.submitContinuation(job); + pub fn submitStep(self: *Scheduler, job: job_system.Job) task_system.TaskSystemError!void { + return self.inner.submitStep(job); } pub fn submitCompute(self: *Scheduler, job: job_system.Job) task_system.TaskSystemError!void { return self.inner.submitCompute(job); } - pub fn continuationJobs(self: *Scheduler) *job_system.JobSystem { - return self.inner.continuationJobs(); + pub fn stepJobs(self: *Scheduler) *job_system.JobSystem { + return self.inner.stepJobs(); } pub fn computeJobs(self: *Scheduler) ?*job_system.JobSystem { diff --git a/src/zerver/runtime/termination.zig b/src/zerver/runtime/termination.zig new file mode 100644 index 0000000..332fb86 --- /dev/null +++ b/src/zerver/runtime/termination.zig @@ -0,0 +1,72 @@ +/// Cross-platform termination signal handling for graceful shutdown logging. +const std = @import("std"); +const builtin = @import("builtin"); +const slog = @import("../observability/slog.zig"); + +const atomic = std.atomic; + +var termination_once = atomic.Value(bool).init(false); + +fn handleTermination(signal_name: []const u8) void { + if (termination_once.swap(true, .seq_cst)) return; + + slog.warn("Termination signal received", &.{ + slog.Attr.string("signal", signal_name), + }); + + // Try to close log file so the entry is flushed. + slog.closeDefaultLoggerFile(); + + // Exit with standard SIGINT status code (130) for Ctrl+C/SIGINT; generic otherwise. + std.process.exit(130); +} + +pub fn installHandlers() !void { + if (builtin.os.tag == .windows) { + return installWindowsHandler(); + } else { + return installPosixHandler(); + } +} + +const windows = std.os.windows; + +fn installWindowsHandler() !void { + try windows.SetConsoleCtrlHandler(consoleCtrlHandler, true); +} + +pub export fn consoleCtrlHandler(ctrl_type: windows.DWORD) callconv(.c) windows.BOOL { + switch (ctrl_type) { + windows.CTRL_C_EVENT => handleTermination("CTRL_C_EVENT"), + windows.CTRL_BREAK_EVENT => handleTermination("CTRL_BREAK_EVENT"), + windows.CTRL_CLOSE_EVENT => handleTermination("CTRL_CLOSE_EVENT"), + windows.CTRL_LOGOFF_EVENT => handleTermination("CTRL_LOGOFF_EVENT"), + windows.CTRL_SHUTDOWN_EVENT => handleTermination("CTRL_SHUTDOWN_EVENT"), + else => return windows.FALSE, + } + + return windows.TRUE; +} + +fn installPosixHandler() !void { + const posix = std.posix; + + var action = posix.Sigaction{ + .handler = .{ .handler = posixSignalHandler }, + .mask = posix.empty_sigset, + .flags = 0, + }; + + try posix.sigaction(posix.SIGINT, &action, null); + try posix.sigaction(posix.SIGTERM, &action, null); +} + +fn posixSignalHandler(sig: c_int) callconv(.C) void { + const posix = std.posix; + const signal_name = switch (sig) { + posix.SIGINT => "SIGINT", + posix.SIGTERM => "SIGTERM", + else => "SIGNAL", + }; + handleTermination(signal_name); +} From 6c44c30a2672b69f06d109d2b28a676790bc41ff Mon Sep 17 00:00:00 2001 From: Earl Cameron Date: Mon, 27 Oct 2025 03:04:48 -0400 Subject: [PATCH 4/4] Implement OpenTelemetry improvements and testing strategy - Added comprehensive OpenTelemetry documentation outlining objectives, span taxonomy, job lifecycle, and sampling strategies. - Introduced OtelConfig struct for managing OpenTelemetry configuration, including environment variable parsing and default values. - Developed a structured testing strategy for the Zerver framework, focusing on HTTP/1.1 RFC compliance with detailed test plans for request line parsing, status line validation, and header field handling. - Created integration tests for various HTTP request scenarios, ensuring correct handling of valid and invalid requests. --- .github/workflows/build.yml | 73 +- build.zig | 1 + config.json | 6 + docs/OTEL_REVISION_PLAN.md | 442 +++++++ docs/OTEL_improvement.md | 530 +++++++++ docs/TESTING_STRATEGY.md | 158 +++ src/features/blog/list.zig | 24 +- src/features/blog/routes.zig | 14 +- src/shared/http.zig | 2 + src/zerver/core/circuit_breaker.zig | 5 + src/zerver/core/core.zig | 3 + src/zerver/core/ctx.zig | 16 +- src/zerver/core/error_renderer.zig | 6 +- src/zerver/core/reqtest.zig | 5 + src/zerver/core/types.zig | 20 +- src/zerver/impure/executor.zig | 77 +- src/zerver/observability/otel.zig | 1024 ++++++++++++----- src/zerver/observability/otel_config.zig | 85 ++ src/zerver/observability/telemetry.zig | 238 ++++ src/zerver/runtime/global.zig | 1 + src/zerver/runtime/handler.zig | 10 + src/zerver/runtime/listener.zig | 3 + .../rfc9112_message_format_test.zig | 130 +++ 23 files changed, 2486 insertions(+), 387 deletions(-) create mode 100644 docs/OTEL_REVISION_PLAN.md create mode 100644 docs/OTEL_improvement.md create mode 100644 docs/TESTING_STRATEGY.md create mode 100644 src/zerver/observability/otel_config.zig create mode 100644 tests/integration/rfc9112_message_format_test.zig diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 641385a..e8f51bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,33 +8,52 @@ on: jobs: build: - runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - + include: + - os: ubuntu-latest + artifact_name: zerver-example-linux + artifact_path: zig-out/bin/zerver_example + - os: macos-latest + artifact_name: zerver-example-macos + artifact_path: zig-out/bin/zerver_example + - os: windows-latest + artifact_name: zerver-example-windows + artifact_path: zig-out/bin/zerver_example.exe + runs-on: ${{ matrix.os }} + steps: - - uses: actions/checkout@v4 - - - name: Cache Zig build cache - id: cache-zig - uses: actions/cache@v4 - with: - path: ~/.zig-cache - key: ${{ runner.os }}-zig-${{ hashFiles('build.zig') }} - restore-keys: | - ${{ runner.os }}-zig- - - - name: Install Zig - uses: ziglang/setup-zig@v1 - with: - zig-version: 0.15.2 - - - name: Build - run: zig build - - - name: Test - run: zig test - - - name: Check formatting - run: zig fmt --check src/ examples/ + - uses: actions/checkout@v4 + + - name: Cache Zig build cache + id: cache-zig + uses: actions/cache@v4 + with: + path: | + zig-cache + ~/.cache/zig + ~/AppData/Local/zig + key: ${{ runner.os }}-zig-${{ hashFiles('build.zig', 'main.zig', 'src/**', 'tests/**', 'examples/**', 'third_party/libuv/**') }} + restore-keys: | + ${{ runner.os }}-zig- + + - name: Install Zig + uses: ziglang/setup-zig@v1 + with: + zig-version: 0.15.2 + + - name: Build (ReleaseSafe) + run: zig build -Doptimize=ReleaseSafe + + - name: Run tests + run: zig build test + + - name: Check formatting + run: zig fmt --check . + + - name: Upload executable + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: ${{ matrix.artifact_path }} + if-no-files-found: error diff --git a/build.zig b/build.zig index bafe32b..fb66a6d 100644 --- a/build.zig +++ b/build.zig @@ -99,6 +99,7 @@ pub fn build(b: *std.Build) void { }); exe.linkLibC(); addLibuv(b, exe); + b.installArtifact(exe); const run_cmd = b.addRunArtifact(exe); run_cmd.step.dependOn(b.getInstallStep()); diff --git a/config.json b/config.json index 1200ade..9a240e0 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,10 @@ { + "project": { + "name": "Zerver", + "version": "0.1.0", + "description": "Zerver is a backend framework for Zig that gives you X-ray vision into your API. It's built on the idea that observability isn't a feature you add later—it's the architecture.", + "repository": "https://github.com/monstercameron/Zerver" + }, "database": { "driver": "sqlite", "path": "resources/blog.db", diff --git a/docs/OTEL_REVISION_PLAN.md b/docs/OTEL_REVISION_PLAN.md new file mode 100644 index 0000000..2f986ee --- /dev/null +++ b/docs/OTEL_REVISION_PLAN.md @@ -0,0 +1,442 @@ +# OpenTelemetry Implementation Revision Plan + +## Executive Summary + +This plan outlines the necessary changes to align the current OTEL implementation with the objectives defined in `OTEL_improvement.md`. The goal is to implement a compact, threshold-based span promotion system that distinguishes logical work from queueing/parking overhead while adhering to OpenTelemetry semantic conventions. + +--- + +## Current State Analysis + +### What Exists + +1. **Core Infrastructure** (`otel.zig`, `telemetry.zig`, `tracer.zig`) + - Full OTLP/HTTP JSON exporter with retry logic + - Request lifecycle tracking with span hierarchy + - Job system event tracking (enqueue, start, complete) + - Subscriber pattern for event distribution + - Resource attributes and instrumentation scope + +2. **Span Hierarchy** + - Root: SERVER span per request + - Steps: INTERNAL spans for each step execution + - Effects: CLIENT spans for I/O operations + - Jobs: INTERNAL spans for effect_job and step_job + +3. **Event Tracking** + - All job lifecycle stages recorded as events + - Events attached to both job spans and parent spans (mirrored) + - Rich metadata: queue name, worker_index, job_ctx, sequences + +4. **Attributes** + - HTTP request/response metadata + - Step/effect sequencing and timing + - Job execution context (queue, worker, success) + - Error context propagation + +### What's Missing/Wrong + +1. **Job Span Behavior** + - Currently: Job spans (`effect_job`, `step_job`) are **always created** as child spans + - Target: Job lifecycle should be **events by default**, promoted to spans only when thresholds exceeded + +2. **Timestamp Tracking** + - Missing: Individual `park_ts[]` and `resume_ts[]` arrays for park episodes + - Missing: `take_ts` (when job dequeued) vs `start_ts` (when work begins) + - Current: Only `start_time_unix_ns` and `end_time_unix_ns` on job spans + +3. **Derived Durations** + - Missing calculation of: + - `queue_wait_ms` (take - enqueue) + - `dispatch_ms` (start - take) + - `park_wait_ms_total` (sum of park episodes) + - `run_active_ms` (total - park_wait) + +4. **Threshold-Based Promotion** + - No environment variable configuration (`ZER_VER_PROMOTE_QUEUE_MS`, etc.) + - No logic to conditionally promote queue/park to spans + - No route-level p95 tracking for dynamic thresholds + +5. **Naming Conventions** + - Root span: Currently `"{method} {path}"` → Should be `"http.server: {method} {route}"` + - Effects: Generic → Should include operation type (e.g., `"db.get blog_post"`) + - Continuations: Not distinguished → Need `"step..resume"` + - Promoted spans: Missing `"job.queue_wait"` and `"job.park()"` naming + +6. **Semantic Conventions Alignment** + - Old keys used: `http.method`, `http.status_code` + - New required: `http.request.method`, `http.response.status_code`, `url.scheme`, `url.path`, etc. + - Missing: `concurrency.limit.*`, `job.park.cause`, `render.*` attributes + +7. **Parking/Concurrency Tracking** + - No park/resume event distinction + - No concurrency limit tracking + - No cause attribution for parking (io_wait, rate_limit, backpressure, etc.) + +8. **Status Code Strategy** + - Current: Status set to ERROR for 500s or failures + - Target: Status should remain UNSET/OK for application-level errors; ERROR reserved for telemetry/system failures + +9. **Rendering Attributes** + - Missing phase tracking (markdown, template, json_encode) + - Missing size_bytes for rendered content + +--- + +## Revision Tasks + +### Phase 1: Configuration & Infrastructure + +**1.1 Add Configuration Module** +- [ ] Create `src/zerver/observability/otel_config.zig` +- [ ] Parse environment variables: + - `ZER_VER_PROMOTE_QUEUE_MS` (default: 5) + - `ZER_VER_PROMOTE_PARK_MS` (default: 5) + - `ZER_VER_DEBUG_JOBS` (default: 0) + - `ZER_VER_QUEUE_NAME_EFFECTS` (default: "effects") + - `ZER_VER_QUEUE_NAME_CONT` (default: "continuations") + - `ZER_VER_EXPORT_JOB_DEPTH` (default: 0) + - `ZER_VER_SAMPLER` (e.g., "head:0.02,tail:error|p99") + - `ZER_VER_METRICS_VIEW_ROUTE_P95_WINDOW` +- [ ] Expose config struct to `OtelExporter.init()` + +**1.2 Enhance RequestRecord with Job Lifecycle State** +- [ ] Add fields to track timestamps: + ```zig + enqueue_ts: ?i64 = null, + take_ts: ?i64 = null, + start_ts: ?i64 = null, + end_ts: ?i64 = null, + park_episodes: std.ArrayList(ParkEpisode), + ``` +- [ ] Define `ParkEpisode`: + ```zig + const ParkEpisode = struct { + cause: []const u8, // io_wait|rate_limit|backpressure|lock|timer|other + token: ?u32, + park_ts: i64, + resume_ts: ?i64, + concurrency_limit_current: ?usize, + concurrency_limit_max: ?usize, + }; + ``` +- [ ] Store per-effect and per-step-job + +**1.3 Add Telemetry Events for Lifecycle Stages** +- [ ] Extend `telemetry.zig` events: + - `EffectJobTaken` (when dequeued, before started) + - `EffectJobParked` (with cause, token, limits) + - `EffectJobResumed` + - `StepJobTaken` + - `StepJobParked` + - `StepJobResumed` +- [ ] Update `Event` union and `RequestRecord.record*` methods + +--- + +### Phase 2: Job Lifecycle Refactoring + +**2.1 Default to Events (Not Spans)** +- [ ] Remove automatic span creation in: + - `recordEffectJobEnqueued` + - `recordStepJobEnqueued` +- [ ] Store lifecycle events as `RequestEvent` on parent span (effect or step) +- [ ] Track job state in a new map: `job_states: AutoHashMap(usize, JobState)` + ```zig + const JobState = struct { + job_type: enum { effect, step }, + sequence: usize, + enqueue_ts: i64, + take_ts: ?i64, + start_ts: ?i64, + end_ts: ?i64, + park_episodes: std.ArrayList(ParkEpisode), + queue: []const u8, + job_ctx: ?usize, + worker_index: ?usize, + success: ?bool, + }; + ``` + +**2.2 Compute Derived Durations** +- [ ] On `recordEffectJobCompleted` / `recordStepJobCompleted`: + - Calculate `queue_wait_ms = take_ts - enqueue_ts` + - Calculate `dispatch_ms = start_ts - take_ts` + - Calculate `park_wait_ms_total = sum(resume_ts[i] - park_ts[i])` + - Calculate `run_active_ms = (end_ts - start_ts) - park_wait_ms_total` +- [ ] Add as attributes to logical span (effect/step) or events + +**2.3 Implement Threshold-Based Promotion** +- [ ] On job completion, check: + - `queue_wait_ms >= config.promote_queue_ms` + - Any `park_wait_ms[i] >= config.promote_park_ms` + - `config.debug_jobs == true` +- [ ] If triggered, create promoted child spans: + - `job.queue_wait` (Kind=INTERNAL) + - `job.park()` (Kind=INTERNAL, one per episode) +- [ ] Attach lifecycle events to promoted spans +- [ ] Finalize derived attributes on promoted spans + +--- + +### Phase 3: Naming & Semantic Conventions + +**3.1 Update Root Span Naming** +- [ ] Change from `"{method} {path}"` to `"http.server: {method} {route}"` +- [ ] Extract route template from path (e.g., `/blogs/posts/{id}`) + +**3.2 Update Step Span Naming** +- [ ] Keep `"step.{name}"` for forward steps +- [ ] Add `"step.{name}.resume"` for continuation resumption +- [ ] Set `Kind=INTERNAL` consistently + +**3.3 Update Effect Span Naming** +- [ ] Include operation: `"db.get blog_post"`, `"http.get comments"`, `"cache.get author"` +- [ ] Extract from effect details (kind + target) +- [ ] Set `Kind=CLIENT` + +**3.4 Adopt New Semantic Conventions** +- [ ] Rename attributes: + - `http.method` → `http.request.method` + - `http.status_code` → `http.response.status_code` + - `http.host` → `server.address` (split port if present) + - `http.user_agent` → `user_agent.original` + - `http.client_ip` → `client.address` +- [ ] Add new attributes: + - `url.scheme`, `url.path`, `server.port`, `client.port` + - `network.transport`, `network.protocol.version` + - `url.path.params.*` (path parameters) +- [ ] Keep old keys for one release (dual-emit) + +**3.5 Add Job-Specific Attributes** +- [ ] On effect/step spans: + - `job.queue_wait_ms`, `job.dispatch_ms`, `job.park_wait_ms_total`, `job.run_active_ms`, `job.park_count` +- [ ] On promoted `job.queue_wait` spans: + - `job.queue`, `job.depth_start`, `job.depth_end` +- [ ] On promoted `job.park` spans: + - `job.park.cause`, `job.park.token`, `concurrency.limit.current`, `concurrency.limit.max` + +**3.6 Add Rendering Attributes** +- [ ] For rendering phases (if detected): + - `render.phase` (markdown, template, json_encode, other) + - `render.size_bytes` + +--- + +### Phase 4: Status Code & Error Strategy + +**4.1 Revise Status Setting Logic** +- [ ] Root span: + - UNSET/OK for `status_code < 500` (even 4xx) + - ERROR only for `status_code >= 500` or internal telemetry failures +- [ ] Step spans: + - OK if decision=Continue/Done + - ERROR only if `outcome="Fail"` AND system-level error (not application logic) +- [ ] Effect spans: + - OK if `success=true` + - ERROR if `required=true AND success=false` AND represents infrastructure failure + - UNSET for non-required failures + +**4.2 Error Context Propagation** +- [ ] Attach `error_type`, `error_message` attributes when available +- [ ] Distinguish application errors from system errors in `ErrorCtx` + +--- + +### Phase 5: Event Refinement + +**5.1 Standardize Event Names** +- [ ] Prefix all custom events with `zerver.` +- [ ] Align names: + - `job.enqueued`, `job.taken`, `job.started`, `job.parked`, `job.resumed`, `job.completed` + - `need.requested`, `need.join` + - `slot.write` + - `retry` + +**5.2 Add New Events** +- [ ] `need.requested {effects:n, mode, join}` (on need schedule) +- [ ] `need.join {completed, failed, duration_ms}` (on need completion) +- [ ] `slot.write {slot, size_bytes}` (on slot writes) +- [ ] `retry {attempt, reason, backoff_ms}` (on effect retries) + +--- + +### Phase 6: Parking & Concurrency + +**6.1 Integrate with Job System** +- [ ] Emit `EffectJobParked` / `StepJobParked` events from job system when: + - Waiting on I/O + - Rate limit hit + - Backpressure applied + - Lock contention + - Timer/sleep +- [ ] Include cause, token, concurrency limits in event +- [ ] Emit `EffectJobResumed` / `StepJobResumed` when work resumes + +**6.2 Track Concurrency Limits** +- [ ] Query job system for current/max concurrency limits per queue +- [ ] Attach to park events and promoted park spans + +--- + +### Phase 7: Sampling & Metrics (Optional) + +**7.1 Head Sampling** +- [ ] Implement probabilistic sampling at root (1–5%) +- [ ] Skip export if sample decision is "drop" + +**7.2 Tail Sampling** +- [ ] Collect triggers: + - `status_code >= 500` + - Duration ≥ route p95/p99 + - Required effect failures + - Queue/park wait ≥ route p95 +- [ ] Force export if any trigger matches + +**7.3 Exemplar Linking** +- [ ] Attach `trace_id` to latency histogram exemplars (future work) + +**7.4 Metrics Derivation** +- [ ] Document exporter-side metrics: + - `http.server.duration_ms`, `zerver.step.duration_ms`, `zerver.effect.duration_ms` + - `zerver.job.queue_wait_ms`, `zerver.job.dispatch_ms`, `zerver.job.park_wait_ms`, `zerver.job.run_active_ms` + - `zerver.job.depth` (gauge) + - `zerver.job.retries_total` + - `cache.hit_ratio` + +--- + +### Phase 8: Testing & Validation + +**8.1 Golden Trace Tests** +- [ ] Create test cases: + - Happy path 200 (no promotions) + - 404 not found (status UNSET/OK, no DB write) + - 500 error (status ERROR) + - Queue-heavy run (queue_wait promoted) + - Park-heavy run (park promoted) + - Parallel effects with join=all +- [ ] Assert: + - Span tree shape (parent/child relationships) + - Required attributes present + - Derived durations consistent with timestamps + - Event sequence validity + +**8.2 Load Testing** +- [ ] Run under load with `ZER_VER_DEBUG_JOBS=1` to verify span creation doesn't degrade performance +- [ ] Test with `ZER_VER_PROMOTE_QUEUE_MS=0` to force all promotions +- [ ] Verify OTLP export succeeds with large span batches + +**8.3 Backwards Compatibility** +- [ ] Dual-emit old and new attribute keys for one release +- [ ] Document migration path in `CHANGELOG.md` + +--- + +## Implementation Order + +### Milestone 1: Job Lifecycle (Weeks 1-2) +1. Add configuration module (1.1) +2. Enhance RequestRecord with job state (1.2) +3. Add new telemetry events (1.3) +4. Refactor to event-first logic (2.1) +5. Compute derived durations (2.2) + +### Milestone 2: Promotion Logic (Week 3) +1. Implement threshold checks (2.3) +2. Create promoted span logic (2.3) +3. Integrate with config (1.1) + +### Milestone 3: Naming & Conventions (Week 4) +1. Update root span naming (3.1) +2. Update step/effect naming (3.2, 3.3) +3. Adopt new semantic conventions (3.4) +4. Add job-specific attributes (3.5) +5. Add rendering attributes (3.6) + +### Milestone 4: Status & Events (Week 5) +1. Revise status logic (4.1, 4.2) +2. Standardize event names (5.1) +3. Add new events (5.2) + +### Milestone 5: Parking & Concurrency (Week 6) +1. Integrate with job system (6.1) +2. Track concurrency limits (6.2) + +### Milestone 6: Testing & Validation (Week 7) +1. Golden trace tests (8.1) +2. Load testing (8.2) +3. Backwards compatibility (8.3) + +### Milestone 7: Sampling & Metrics (Optional, Week 8+) +1. Head sampling (7.1) +2. Tail sampling (7.2) +3. Exemplar linking (7.3) +4. Metrics derivation (7.4) + +--- + +## Key Design Decisions + +### 1. **Event-First, Span-on-Threshold** +- **Rationale**: Keeps default traces compact; deep visibility only when latency indicates issues +- **Trade-off**: Adds complexity to promotion logic but dramatically reduces span count + +### 2. **Park Episodes as Array** +- **Rationale**: Jobs can park multiple times (I/O wait, then rate limit, etc.) +- **Trade-off**: Requires dynamic allocation but captures full concurrency story + +### 3. **Dual-Emit for Migration** +- **Rationale**: Allows downstream consumers to migrate gradually +- **Trade-off**: Temporary attribute bloat, removed in next major version + +### 4. **Status=UNSET for Application Errors** +- **Rationale**: Aligns with OTEL philosophy (status is for telemetry health, not app logic) +- **Trade-off**: May confuse users expecting ERROR for 404/400; document clearly + +### 5. **Route-Level p95 Thresholds (Future)** +- **Rationale**: Static thresholds don't adapt to varying route latencies +- **Trade-off**: Requires aggregation infra; defer to Phase 7 or later + +--- + +## Risk Mitigation + +1. **Breaking Changes**: Dual-emit old keys for one release; document migration +2. **Performance Regression**: Profile promotion logic; ensure O(1) threshold checks +3. **Memory Leaks**: Carefully manage `ParkEpisode` allocations; use arena for temp data +4. **Job System Integration**: Coordinate with reactor team on park/resume event emission +5. **Testing Gaps**: Require golden traces before merging any naming/convention changes + +--- + +## Success Criteria + +- [ ] Job lifecycle fully captured with enqueue/take/start/park/resume/complete timestamps +- [ ] Derived durations (queue_wait, dispatch, park_wait, run_active) computed correctly +- [ ] Promotion logic triggered by thresholds; `ZER_VER_DEBUG_JOBS=1` forces promotion +- [ ] Span naming matches spec: `http.server: GET /path`, `db.get table`, `step.name.resume` +- [ ] New semantic conventions adopted; old keys dual-emitted +- [ ] Status codes set per policy (UNSET for app errors, ERROR for system failures) +- [ ] Golden trace tests pass; load tests show no perf degradation +- [ ] Backwards compatibility verified + +--- + +## Next Steps + +1. **Review with team**: Validate approach, especially job system integration points +2. **Prototype Milestone 1**: Build config + job state tracking in isolation +3. **Integration testing**: Coordinate with reactor team for park/resume events +4. **Iterate on naming**: Confirm naming conventions with OTEL community best practices +5. **Document**: Update `API_REFERENCE.md` and `OTEL_improvement.md` with final design + +--- + +## References + +- `docs/OTEL_improvement.md` (requirements) +- `src/zerver/observability/otel.zig` (current implementation) +- `src/zerver/observability/telemetry.zig` (event definitions) +- `src/zerver/runtime/reactor/job_system.zig` (job lifecycle) +- OpenTelemetry Semantic Conventions: https://opentelemetry.io/docs/specs/semconv/ diff --git a/docs/OTEL_improvement.md b/docs/OTEL_improvement.md new file mode 100644 index 0000000..a109974 --- /dev/null +++ b/docs/OTEL_improvement.md @@ -0,0 +1,530 @@ +# Objectives + +* Distinguish logical work, queueing, and parking latencies. +* Keep default traces compact; auto-promote when thresholds are exceeded. +* Align with current OpenTelemetry semantic conventions. + +# Span Taxonomy + +* Root: `http.server` (Kind=SERVER). +* Steps: `step.*` (Kind=INTERNAL). +* Effects (I/O): Kind=CLIENT (e.g., DB/HTTP/cache). +* Job internals (queue/park) promoted spans: `job.queue_wait`, `job.park` (Kind=INTERNAL). + +# Job Lifecycle + +State machine for any internal job (`effect` or `continuation`): + +``` +ENQUEUE → TAKE → START → [PARK ↔ RESUME]* → COMPLETE | FAIL | CANCEL +``` + +Timestamps: + +* `enqueue_ts`, `take_ts`, `start_ts`, `end_ts` +* `park_ts[i]`, `resume_ts[i]` for i∈[0..n) + +Derived durations: + +* `queue_wait_ms = take_ts - enqueue_ts` +* `dispatch_ms = start_ts - take_ts` +* `park_wait_ms_total = Σ(resume_ts[i] - park_ts[i])` +* `run_active_ms = (end_ts - start_ts) - park_wait_ms_total` + +# Default Mode (Compact) + +* One span per logical unit: + + * Effect → e.g., `db.get blog_post` (Kind=CLIENT). + * Continuation → `step..resume` (Kind=INTERNAL). +* Emit lifecycle **events** on the logical span: + + * `job.enqueued {queue, depth_start}` + * `job.taken {worker_id}` + * `job.started` + * `job.parked {cause, token, concurrency.limit}` + * `job.resumed` + * `job.completed {success, attempts}` + * `job.failed {error_type, error_message?}` +* Finalize **attributes** on completion (see “Attributes”). + +# Expanded Mode (Threshold/Debug) + +Auto-promote queue/park to spans (children of the logical span) when any condition holds: + +* `queue_wait_ms >= ZER_VER_PROMOTE_QUEUE_MS` OR ≥ route p95. +* Any single park episode ≥ `ZER_VER_PROMOTE_PARK_MS` OR ≥ route p95. +* `ZER_VER_DEBUG_JOBS=1`. + +Promoted spans: + +* `job.queue_wait` (Kind=INTERNAL). + + * Attributes: `job.queue`, `job.depth_start`, `job.depth_end` (if known). +* `job.park` (Kind=INTERNAL), one per episode promoted. + + * Attributes: `job.park.cause = io_wait|rate_limit|backpressure|lock|timer|other`, + `job.park.token`, `concurrency.limit.current`, `concurrency.limit.max`. + +Events remain on the logical span. + +# Naming + +* Root: `http.server: GET /blogs/posts/{id}` +* Steps: `step.route_match`, `step.load_blog_post_page` +* Effects: `db.get blog_post`, `http.get comments`, `cache.get blog_post` +* Continuations: `step.load_blog_post_page.resume` +* Job internals (promoted): `job.queue_wait`, `job.park()` + +# Attributes + +## Root (SERVER) + +* `http.request.method` +* `http.route` +* `url.scheme`, `url.path`, `server.address`, `server.port` +* `client.address`, `client.port` (if available) +* `user_agent.original` +* `network.transport`, `network.protocol.version` +* `url.path.params.id` +* `zerver.request_id`, `session.id` (if available) +* `http.response.status_code` +* `http.request.body.size`, `http.response.body.size` +* Status set only on error (else UNSET or OK per policy). + +## Step Spans (INTERNAL) + +* `zerver.step.sequence` (int) +* `zerver.step.layer` (int or label) +* `zerver.step.decision = Need|Continue|Done` +* `zerver.step.reads = [""]` +* `zerver.step.writes = [""]` +* `zerver.step.resume = ""` (if decision=Need/Continue) +* Optional: `validation.errors_count` + +## Effect Spans (CLIENT) + +* Common: + + * `effect.kind = db|http|cache|kv|queue|fs|other` + * `effect.required = true|false` + * `effect.timeout_ms` + * `job.queue = "effects"`, `job.worker_id`, `job.attempt` + * `job.queue_wait_ms`, `job.dispatch_ms`, `job.park_wait_ms_total`, `job.run_active_ms`, `job.park_count` +* DB: + + * `db.system` (postgres|mysql|sqlite|redis|kv) + * `db.operation` (SELECT|GET|INSERT|UPDATE|DELETE) + * `db.namespace` (table/collection) + * `server.address`, `server.port` (DB peer) + * `db.key` (sanitized), `cache.hit` (if layered) +* HTTP: + + * `url.full` (sanitized), `server.address`, `server.port` + * `http.request.method`, `http.response.status_code` +* Cache/KV: + + * `cache.system`, `cache.operation`, `cache.hit` + +## Continuation Spans (INTERNAL) + +* `zerver.resume.fn = ""` +* `zerver.decision = Done|Need` (post-resume) +* Rendering: + + * `render.phase = markdown|template|json_encode|other` + * `render.size_bytes` + +## Promoted `job.queue_wait` Span (INTERNAL) + +* `job.queue`, `job.depth_start`, `job.depth_end` + +## Promoted `job.park` Span (INTERNAL) + +* `job.park.cause = io_wait|rate_limit|backpressure|lock|timer|other` +* `job.park.token` +* `concurrency.limit.current`, `concurrency.limit.max` + +## Resource Attributes + +* `service.name`, `service.version`, `service.instance.id` +* `deployment.environment` +* `host.name` +* `process.pid` +* `process.runtime.name = "zig"`, `process.runtime.version` +* `telemetry.sdk.name`, `telemetry.sdk.version` + +# Events (All Modes; on Logical Spans) + +* `job.enqueued {queue, depth}` +* `job.taken {worker_id}` +* `job.started` +* `job.parked {cause, token, concurrency.limit}` +* `job.resumed` +* `job.completed {success, attempts}` +* `retry {attempt, reason, backoff_ms}` +* `need.requested {effects:n, mode:Sequential|Parallel, join:all|any}` +* `need.join {completed, failed, duration_ms}` +* `slot.write {slot, size_bytes}` + +# Sampling + +* **Head sampling** at root: 1–5%. +* **Tail sampling** triggers: + + * `http.response.status_code >= 500` + * total duration ≥ route p95/p99 + * `effect.required=true AND success=false` + * `job.queue_wait_ms ≥ route p95` OR `job.park_wait_ms_total ≥ route p95` +* Exemplars: attach trace IDs to latency histograms. + +# Metrics (Exporter-Derived) + +Histograms/counters with labels: + +* `http.server.duration_ms{route,status_class}` +* `zerver.step.duration_ms{step}` +* `zerver.effect.duration_ms{effect.kind,operation,required,success}` +* `zerver.job.queue_wait_ms{queue,route,job.type}` +* `zerver.job.dispatch_ms{queue,route,job.type}` +* `zerver.job.park_wait_ms{queue,route,job.type,cause}` +* `zerver.job.run_active_ms{queue,route,job.type}` +* `zerver.job.depth{queue}` (gauge) +* `zerver.job.retries_total{route,job.type}` +* `cache.hit_ratio{namespace}` (if applicable) + +# Exporter Mapping + +* Root: `SpanKind.SERVER` +* Steps/Continuation: `SpanKind.INTERNAL` +* Effects: `SpanKind.CLIENT` +* Promoted job internals: `SpanKind.INTERNAL` +* OTLP/HTTP export; keep current attributes, add new semconv keys; transitional dual-write allowed. + +# Configuration (Env) + +* `ZER_VER_PROMOTE_QUEUE_MS` (default `5`) +* `ZER_VER_PROMOTE_PARK_MS` (default `5`) +* `ZER_VER_DEBUG_JOBS` (`0|1`) +* `ZER_VER_QUEUE_NAME_EFFECTS` (default `"effects"`) +* `ZER_VER_QUEUE_NAME_CONT` (default `"continuations"`) +* `ZER_VER_EXPORT_JOB_DEPTH` (`0|1`) +* `ZER_VER_SAMPLER` (e.g., `head:0.02,tail:error|p99`) +* `ZER_VER_METRICS_VIEW_ROUTE_P95_WINDOW` (aggregation window) + +# Trace Shapes (Examples) + +## Fast path (no promotions) + +``` +SERVER http.server: GET /blogs/posts/{id} + INTERNAL step.route_match + INTERNAL step.load_blog_post_page (decision=Need) + CLIENT db.get blog_post + INTERNAL step.load_blog_post_page.resume +``` + +## Slow effect queue + park (promotions enabled) + +``` +SERVER http.server: GET /blogs/posts/{id} + INTERNAL step.route_match + INTERNAL step.load_blog_post_page (decision=Need) + CLIENT db.get blog_post + INTERNAL job.queue_wait # ≥ threshold + INTERNAL job.park(io_wait, db_pool) + INTERNAL step.load_blog_post_page.resume +``` + +## Parallel effects with join=all + +``` +SERVER http.server: GET /blogs/posts/{id} + INTERNAL step.prepare_post (decision=Need, mode=Parallel, join=all) + CLIENT db.get blog_post + INTERNAL job.queue_wait # if promoted + CLIENT cache.get author_profile + INTERNAL step.prepare_post.resume +``` + +# Testing + +* Golden-trace tests for: + + * Happy path 200 + * 404 not found (no DB write; status UNSET/OK, app status 404) + * DB error (effect.required=true → 500) + * Queue-heavy run (queue_wait promoted) + * Park-heavy run (park promoted) +* Assertions: + + * Span names/kinds tree shape + * Required attributes present + * Derived durations computed and consistent with timestamps + * Events sequence validity + +# Backwards Compatibility + +* Dual-emit old keys (`http.method`, `http.status_code`) alongside new (`http.request.method`, `http.response.status_code`) for one release. +* Keep `effect_job` as events only; re-enable as spans with `ZER_VER_DEBUG_JOBS=1`. + +--- + +# Implementation Notes + +## Architecture Overview + +The OTEL implementation was refactored to use an **event-first architecture** with **threshold-based span promotion**. This approach significantly reduces trace overhead for fast operations while maintaining full observability for slow or problematic jobs. + +### Key Design Decisions + +**1. Event-First Job Tracking** + +Instead of creating spans immediately when jobs are enqueued, the system now: +- Creates a lightweight `JobState` struct to track lifecycle timestamps +- Records lifecycle events on parent spans (effect/step/root) +- Only promotes to full spans when thresholds are exceeded + +**Rationale**: Creating spans for every job incurs allocation and export overhead. Most jobs complete quickly (< 5ms) and don't need dedicated spans. Events provide sufficient visibility without the cost. + +**2. 5ms Default Thresholds** + +Default promotion thresholds: +- `ZER_VER_PROMOTE_QUEUE_MS = 5` +- `ZER_VER_PROMOTE_PARK_MS = 5` + +**Rationale**: +- 5ms represents ~10% of a typical 50ms P50 request latency +- Queue wait > 5ms indicates contention worth investigating +- Park wait > 5ms suggests I/O bottlenecks or rate limiting +- Threshold is low enough to catch issues but high enough to avoid span explosion +- User-configurable via environment variables for different use cases + +**3. JobState vs Immediate Spans** + +`JobState` structure stores: +```zig +struct JobState { + enqueue_ts: i64, + take_ts: ?i64, + start_ts: ?i64, + end_ts: ?i64, + park_episodes: ArrayList(ParkEpisode), + queue: []const u8, + job_type: enum { effect, step }, + // ... other metadata +} +``` + +**Rationale**: +- Memory efficient: ~100 bytes vs ~1KB+ for spans +- Fast allocation: stack-friendly struct vs heap-allocated span tree +- Flexible: can compute durations and decide on promotion at completion +- Clean separation: state tracking vs observability output + +**4. ParkEpisode Tracking** + +Multiple parking events per job tracked as array: +```zig +struct ParkEpisode { + cause: []const u8, // io_wait|rate_limit|backpressure|lock|timer + token: ?u32, + park_ts: i64, + resume_ts: ?i64, + concurrency_limit_current: ?usize, + concurrency_limit_max: ?usize, +} +``` + +**Rationale**: +- Jobs can park multiple times (DB connection pool, rate limiter, etc.) +- Each episode needs cause attribution for debugging +- Token enables matching park/resume pairs in concurrent scenarios +- Concurrency limits help diagnose resource exhaustion +- Total park time = sum of all episodes + +**5. Backfill Event Strategy** + +When a job is promoted to a span, `backfillJobEvents()` reconstructs the complete timeline: +``` +enqueue → taken → started → [parked/resumed]* → completed +``` + +**Rationale**: +- Promoted spans should have complete lifecycle for debugging +- Events provide exact timestamps without span overhead during execution +- Backfilling is one-time cost at promotion (rare for fast jobs) +- Maintains event ordering and causality +- Enables timeline visualization in observability tools + +**6. Threshold Evaluation** + +Promotion happens on job completion when: +```zig +queue_wait_ms >= 5 OR park_wait_ms_total >= 5 OR debug_enabled +``` + +**Rationale**: +- Can't evaluate threshold until job completes (need all timestamps) +- Either queue or park slowness is worth investigating +- Debug mode (`ZER_VER_DEBUG_JOBS=1`) forces promotion for development +- OR logic means any bottleneck triggers promotion +- Short-circuits: if queue wait exceeds threshold, span is promoted regardless of park time + +**7. Attribute Naming Conventions** + +Aligned with OTEL v1.x semantic conventions: +- `http.request.method` (was `http.method`) +- `http.response.status_code` (was `http.status_code`) +- `job.effect.sequence` (was `job.effect_sequence`) +- `job.step.ctx` (was `job.ctx`) +- `effect.sequence` (was `job.need_sequence` in effect context) +- `need.sequence` (was `job.need_sequence` in step context) + +**Rationale**: OTEL v1.x uses hierarchical dot notation for namespacing. Improves compatibility with observability tools and query patterns. + +## Implementation Details + +### Data Structures + +**JobState HashMap**: `std.AutoHashMap(usize, JobState)` +- Key: `effect_sequence` for effect jobs, `job_ctx` for step jobs +- Stored in `RequestRecord` alongside spans +- Cleaned up after job completion or promotion + +**Duration Computation**: +```zig +struct JobDurations { + queue_wait_ms: i64, // take_ts - enqueue_ts + dispatch_ms: i64, // start_ts - take_ts + park_wait_ms_total: i64, // sum(resume_ts - park_ts) + run_active_ms: i64, // (end_ts - start_ts) - park_wait_ms_total + total_ms: i64, // end_ts - enqueue_ts +} +``` + +**Rationale**: Pre-computed durations enable consistent threshold checks and span attributes. Handles missing timestamps gracefully (returns 0 for incomplete phases). + +### Span Management + +**Before**: `ensureJobSpan()` created spans eagerly on enqueue +**After**: Spans created conditionally in `recordEffectJobCompleted()` / `recordStepJobCompleted()` + +**Lifecycle**: +1. Enqueue: Create `JobState`, record event on parent +2. Taken: Update `JobState.take_ts`, record event +3. Started: Update `JobState.start_ts`, record event +4. Parked/Resumed: Append `ParkEpisode`, record events +5. Completed: Compute durations, check thresholds, promote if needed, backfill events + +**Span Hierarchy**: +- Effect job spans → parent to effect span +- Step job spans → parent to root span +- Promoted job spans inherit parent's span_id + +### Event Emission + +**Events vs Spans Decision Matrix**: +| Job Duration | Queue Wait | Park Wait | Result | +|--------------|------------|-----------|--------| +| < 5ms total | < 5ms | < 5ms | Events only (default) | +| ≥ 5ms total | ≥ 5ms | < 5ms | Promoted span + backfilled events | +| ≥ 5ms total | < 5ms | ≥ 5ms | Promoted span + backfilled events | +| Any | Any | Any | Promoted if debug enabled | + +**Event Attributes** (all jobs): +- `job.type`: "effect" or "step" +- `job.queue`: queue name +- `job.stage`: enqueued|taken|started|parked|resumed|completed +- `effect.sequence` or `need.sequence`: parent identifier +- `job.effect.sequence` or `job.step.ctx`: job identifier + +**Span Attributes** (promoted jobs only): +- All event attributes plus: +- `job.queue_wait_ms`, `job.dispatch_ms`, `job.park_wait_ms_total`, `job.run_active_ms`, `job.total_ms` +- `job.park_count`: number of parking episodes +- `job.worker_index`: which worker executed the job +- `job.success`: boolean completion status + +## Performance Characteristics + +**Memory Savings**: +- JobState: ~100 bytes +- Span with events: ~1-2 KB +- **Reduction**: ~90% for fast jobs (no promotion) + +**CPU Savings**: +- No span ID generation for fast jobs +- No event serialization until export (not per-lifecycle-stage) +- Fewer allocations per request + +**Trace Volume**: +- **Before**: Every job created a span (potentially 10-100 per request) +- **After**: Only slow jobs create spans (typically 0-5 per request) +- **Reduction**: 80-95% for typical workloads + +**Trade-offs**: +- Added complexity: JobState management +- Delayed promotion: Can't see span until job completes +- Memory held longer: JobState lives until completion vs spans exported incrementally + +## Configuration + +Environment variables (parsed in `src/zerver/observability/otel_config.zig`): + +```bash +# Queue wait threshold (milliseconds) +export ZER_VER_PROMOTE_QUEUE_MS=5 + +# Park wait threshold (milliseconds) +export ZER_VER_PROMOTE_PARK_MS=5 + +# Force promotion of all jobs (debug) +export ZER_VER_DEBUG_JOBS=1 +``` + +**Use Cases**: +- **Production**: Default thresholds (5ms) for balanced observability +- **Performance Testing**: Set thresholds to 100ms+ to minimize overhead +- **Debugging**: Set `ZER_VER_DEBUG_JOBS=1` for full visibility +- **High-Throughput APIs**: Set thresholds to 10-20ms to reduce trace volume + +## Migration Path + +**Phase 1** (Current): Event-first with threshold promotion +- JobState tracking implemented +- Backfill events on promotion +- HTTP semantic conventions updated + +**Phase 2** (TODO): Runtime integration +- Wire `telemetry.effectJobTaken()` / `stepJobTaken()` in job system +- Add park/resume calls in I/O wait paths + +**Phase 3** (TODO): Validation +- Integration tests for event-first recording +- Integration tests for threshold promotion +- Unit tests for duration computation +- End-to-end trace verification + +**Phase 4** (Future): Adaptive thresholds +- Replace hardcoded 5ms with per-route P95 +- Dynamic adjustment based on traffic patterns +- Tail-based sampling integration + +## Lessons Learned + +1. **State-first, spans later**: Deferring span creation until necessary reduces overhead dramatically +2. **Thresholds matter**: 5ms captures real issues without span explosion +3. **Events are cheap**: Use events for high-frequency data, spans for aggregation +4. **Backfilling works**: Reconstructing timelines from state is feasible and clean +5. **Semantic conventions evolve**: Namespacing (dots) is future-proof +6. **Debug modes essential**: Production optimization shouldn't block development visibility + +## Future Enhancements + +1. **Adaptive Thresholds**: Use per-route P95/P99 instead of fixed 5ms +2. **Tail Sampling**: Promote spans for error requests regardless of duration +3. **Exemplar Linking**: Attach trace IDs to duration histograms +4. **Queue Depth Tracking**: Add `job.depth_start` and `job.depth_end` attributes +5. **Worker Pool Metrics**: Track utilization and contention +6. **Concurrency Limits**: Expose limit exhaustion as events/attributes diff --git a/docs/TESTING_STRATEGY.md b/docs/TESTING_STRATEGY.md new file mode 100644 index 0000000..503579b --- /dev/null +++ b/docs/TESTING_STRATEGY.md @@ -0,0 +1,158 @@ +# Zerver Test Suite Strategy + +This document outlines the strategy and structure for the Zerver test suite. The goal is to ensure the framework is robust, reliable, and compliant with web standards, drawing inspiration from the comprehensive testing practices of mature frameworks like Express.js. + +## 1. Philosophy and Goals + +- **Correctness First:** The primary goal is to ensure Zerver behaves exactly as documented and is compliant with relevant RFCs (HTTP, URI, etc.). +- **Low-Level and High-Level Testing:** The suite will include both low-level raw HTTP/1.1 text-based tests and high-level integration tests that verify application-level behavior. +- **Clarity and Readability:** Tests should be easy to read and understand, serving as a form of living documentation for the framework's behavior. +- **Performance:** While correctness is the priority, the test suite should be designed to run efficiently to facilitate rapid development cycles. +- **Automation:** All tests must be fully automated and runnable with a single command (`zig build test`). + +## 2. Test Harness and Tools + +- **`reqtest.zig`:** This is the core of our testing strategy. It allows us to create and dispatch raw HTTP/1.1 requests as text and assert against the raw text of the response. This is a key differentiator from other frameworks and is essential for ensuring RFC compliance. +- **`std.testing`:** Zig's standard testing library will be used for all test assertions. +- **`main.zig` and other examples:** The example applications will be used as the basis for integration and acceptance tests. + +## 3. Test Organization + +The test suite will be organized into the following directories under the `tests/` directory: + +- **`pure/`:** Tests for pure functions and data structures that do not involve I/O. +- **`unit/`:** Unit tests for individual components and modules in isolation. +- **`integration/`:** Tests that verify the interaction between multiple components, such as the router and the server. +- **`acceptance/`:** High-level tests that verify the behavior of the entire framework from the perspective of a client. These tests will be based on the example applications. +- **`perf/`:** Performance and benchmark tests. + +## 4. HTTP/1.1 RFC Compliance Test Plan + +Achieving 100% coverage of the HTTP/1.1 RFCs is a primary goal for Zerver. This section outlines a detailed test plan to verify compliance with RFC 9110 (HTTP Semantics) and RFC 9112 (HTTP/1.1). All tests in this section must be implemented using the `reqtest.zig` raw text harness. + +### 4.1. RFC 9112 - HTTP/1.1 Message Format + +#### 4.1.1. Section 2: Message Parsing + +- **[2.1] Request Line:** + - **Positive:** + - Test with all standard methods (GET, POST, PUT, DELETE, HEAD, OPTIONS, TRACE, CONNECT). + - Test with various valid paths: `/`, `/foo`, `/foo/bar`, `/foo/bar/`, `/foo%20bar`. + - Test with `HTTP/1.1`. + - **Negative:** + - Test with an invalid method (e.g., `INVALID`). + - Test with a missing method, path, or version. + - Test with an invalid version (e.g., `HTTP/1.2`, `HTTP/1.0`). + - Test with a non-HTTP protocol (e.g., `FTP/1.0`). + - **Edge Cases:** + - Test with extra whitespace before, between, and after elements. + - Test with a very long path (e.g., 8000+ characters). + - Test with a request line containing only whitespace. + +- **[2.2] Status Line:** + - **Positive:** + - Test with a representative set of valid status codes (200, 201, 204, 301, 302, 400, 404, 500). + - **Negative:** + - Test with a missing version, status code, or reason phrase. + - Test with an invalid version or status code (e.g., 99, 600). + - Test with a non-numeric status code. + +- **[2.3] Header Fields:** + - **Positive:** + - Test with single and multiple header fields. + - Test with case-insensitive header field names (e.g., `Host`, `host`, `HOST`). + - Test with header values that are quoted strings. + - **Negative:** + - Test with header fields containing invalid characters (e.g., control characters, non-ASCII). + - Test with a missing colon separator. + - Test with a header field name that is not a token. + - **Edge Cases:** + - Test with header fields containing obsolete line folding (a CRLF followed by a space or tab). + - Test with very long header fields (e.g., 8000+ characters). + - Test with multiple headers of the same name, and verify they are correctly combined. + +#### 4.1.2. Section 3: Message Body + +- **[3.2] Content-Length:** + - **Positive:** + - Test with a valid `Content-Length` header. + - Test with a `Content-Length` of 0. + - **Negative:** + - Test with an invalid `Content-Length` (e.g., non-numeric, negative). + - Test with multiple `Content-Length` headers with different values (should be rejected). + - Test with a `Content-Length` header that is larger than the actual body. + - Test with a `Content-Length` header that is smaller than the actual body. +- **[3.3] Message Body Length:** + - **Positive:** + - Test with a message body that matches the `Content-Length`. + - **Negative:** + - Test with a request that has a body but no `Content-Length` or `Transfer-Encoding` (should be rejected). + +#### 4.1.3. Section 4: Chunked Transfer Coding + +- **[4.1] Chunked Body:** + - **Positive:** + - Test with a single chunk. + - Test with multiple chunks. + - Test with a zero-length chunk indicating the end of the body. + - Test with chunk extensions. + - Test with trailer fields. + - **Negative:** + - Test with a malformed chunk size (e.g., non-hex, negative). + - Test with a chunk size that doesn't match the chunk data. + - Test with a missing `0` chunk at the end. + - Test with data after the last chunk. + +#### 4.1.4. Section 5: Control Data + +- **[5.1] Host and :authority:** + - **Positive:** + - Test with a valid `Host` header. + - **Negative:** + - Test with a missing `Host` header (should result in a 400 Bad Request). + - Test with multiple `Host` headers (should result in a 400 Bad Request). + +#### 4.1.5. Section 6: Connection Management + +- **[6.1] Connection:** + - **Positive:** + - Test with `Connection: keep-alive`. + - Test with `Connection: close`. + - Test with multiple connection options (e.g., `keep-alive, upgrade`). + - **Negative:** + - Test with an invalid `Connection` header value. + +### 4.2. RFC 9110 - HTTP Semantics + +#### 4.2.1. Section 9: Methods + +- **[9.3.1] GET:** + - **Positive:** Test a simple GET request. +- **[9.3.2] HEAD:** + - **Positive:** Test that a HEAD request returns the same headers as a GET request, but with no body. +- **[9.3.3] POST:** + - **Positive:** Test a simple POST request with a body. +- **[9.3.4] PUT:** + - **Positive:** Test a simple PUT request with a body. +- **[9.3.5] DELETE:** + - **Positive:** Test a simple DELETE request. +- **[9.3.6] CONNECT:** + - **Positive:** Test a CONNECT request to establish a tunnel. +- **[9.3.7] OPTIONS:** + - **Positive:** Test an `OPTIONS *` request. + - **Positive:** Test an `OPTIONS` request for a specific resource. +- **[9.3.8] TRACE:** + - **Positive:** Test a TRACE request. + +#### 4.2.2. Section 15: Status Codes + +- **[15.2] 1xx (Informational):** + - **Positive:** Test `100 Continue`. +- **[15.3] 2xx (Successful):** + - **Positive:** Test `200 OK`, `201 Created`, `204 No Content`. +- **[15.4] 3xx (Redirection):** + - **Positive:** Test `301 Moved Permanently`, `302 Found`, `304 Not Modified`, `307 Temporary Redirect`. +- **[15.5] 4xx (Client Error):** + - **Positive:** Test `400 Bad Request`, `401 Unauthorized`, `403 Forbidden`, `404 Not Found`, `405 Method Not Allowed`. +- **[15.6] 5xx (Server Error):** + - **Positive:** Test `500 Internal Server Error`, `501 Not Implemented`, `503 Service Unavailable`. diff --git a/src/features/blog/list.zig b/src/features/blog/list.zig index e9c160d..35777a6 100644 --- a/src/features/blog/list.zig +++ b/src/features/blog/list.zig @@ -194,7 +194,11 @@ pub fn step_load_blog_posts(ctx: *zerver.CtxBase) !zerver.Decision { const effects = try util.singleEffect(ctx, .{ .db_get = .{ .key = "posts", .token = slotId(.PostList), .required = true }, }); - return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all, .continuation = continuation_render_blog_list_page } }; + return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all } }; +} + +pub fn step_render_blog_list_page(ctx: *zerver.CtxBase) !zerver.Decision { + return continuation_render_blog_list_page(ctx); } fn continuation_render_blog_list_page(ctx: *zerver.CtxBase) !zerver.Decision { @@ -249,7 +253,11 @@ pub fn step_load_blog_post_cards(ctx: *zerver.CtxBase) !zerver.Decision { const effects = try util.singleEffect(ctx, .{ .db_get = .{ .key = "posts", .token = slotId(.PostList), .required = true }, }); - return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all, .continuation = continuation_render_blog_post_cards } }; + return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all } }; +} + +pub fn step_render_blog_post_cards(ctx: *zerver.CtxBase) !zerver.Decision { + return continuation_render_blog_post_cards(ctx); } fn continuation_render_blog_post_cards(ctx: *zerver.CtxBase) !zerver.Decision { @@ -280,7 +288,11 @@ pub fn step_load_single_blog_post_card(ctx: *zerver.CtxBase) !zerver.Decision { const effects = try util.singleEffect(ctx, .{ .db_get = .{ .key = effect_key, .token = slotId(.PostJson), .required = true }, }); - return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all, .continuation = continuation_render_single_blog_post_card } }; + return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all } }; +} + +pub fn step_render_single_blog_post_card(ctx: *zerver.CtxBase) !zerver.Decision { + return continuation_render_single_blog_post_card(ctx); } fn continuation_render_single_blog_post_card(ctx: *zerver.CtxBase) !zerver.Decision { @@ -349,7 +361,11 @@ pub fn step_load_blog_post_page(ctx: *zerver.CtxBase) !zerver.Decision { const effects = try util.singleEffect(ctx, .{ .db_get = .{ .key = effect_key, .token = slotId(.PostJson), .required = true }, }); - return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all, .continuation = continuation_render_blog_post_page } }; + return .{ .need = .{ .effects = effects, .mode = .Sequential, .join = .all } }; +} + +pub fn step_render_blog_post_page(ctx: *zerver.CtxBase) !zerver.Decision { + return continuation_render_blog_post_page(ctx); } fn continuation_render_blog_post_page(ctx: *zerver.CtxBase) !zerver.Decision { diff --git a/src/features/blog/routes.zig b/src/features/blog/routes.zig index d5cbbb6..52f5ab1 100644 --- a/src/features/blog/routes.zig +++ b/src/features/blog/routes.zig @@ -27,10 +27,14 @@ const homepage_step = zerver.step("homepage", page.homepageStep); // Blog list page steps const load_blog_posts_step = zerver.step("load_blog_posts", list.step_load_blog_posts); +const render_blog_list_page_step = zerver.step("render_blog_list_page", list.step_render_blog_list_page); const load_blog_post_cards_step = zerver.step("load_blog_post_cards", list.step_load_blog_post_cards); +const render_blog_post_cards_step = zerver.step("render_blog_post_cards", list.step_render_blog_post_cards); const load_single_blog_post_card_step = zerver.step("load_single_blog_post_card", list.step_load_single_blog_post_card); +const render_single_blog_post_card_step = zerver.step("render_single_blog_post_card", list.step_render_single_blog_post_card); const render_blog_list_header_step = zerver.step("render_blog_list_header", list.step_render_blog_list_header); const load_blog_post_page_step = zerver.step("load_blog_post_page", list.step_load_blog_post_page); +const render_blog_post_page_step = zerver.step("render_blog_post_page", list.step_render_blog_post_page); pub fn registerRoutes(srv: *zerver.Server) !void { // Homepage route @@ -40,16 +44,16 @@ pub fn registerRoutes(srv: *zerver.Server) !void { // Blog list page with full HTML try srv.addRoute(.GET, "/blogs/list", .{ - .steps = &.{load_blog_posts_step}, + .steps = &.{ load_blog_posts_step, render_blog_list_page_step }, }); // HTMX fragment endpoints try srv.addRoute(.GET, "/blogs/htmx/cards", .{ - .steps = &.{load_blog_post_cards_step}, + .steps = &.{ load_blog_post_cards_step, render_blog_post_cards_step }, }); try srv.addRoute(.GET, "/blogs/htmx/card/:id", .{ - .steps = &.{load_single_blog_post_card_step}, + .steps = &.{ load_single_blog_post_card_step, render_single_blog_post_card_step }, }); try srv.addRoute(.GET, "/blogs/htmx/header", .{ @@ -58,10 +62,10 @@ pub fn registerRoutes(srv: *zerver.Server) !void { // Blog post page try srv.addRoute(.GET, "/blogs/posts/:id", .{ - .steps = &.{load_blog_post_page_step}, + .steps = &.{ load_blog_post_page_step, render_blog_post_page_step }, }); try srv.addRoute(.GET, "/blogs/posts/:id/fragment", .{ - .steps = &.{load_blog_post_page_step}, + .steps = &.{ load_blog_post_page_step, render_blog_post_page_step }, }); // Posts API diff --git a/src/shared/http.zig b/src/shared/http.zig index 028a17b..e834ef8 100644 --- a/src/shared/http.zig +++ b/src/shared/http.zig @@ -5,6 +5,8 @@ const JSON_HEADERS = [_]zerver.types.Header{ .{ .name = "Cache-Control", .value = "no-store" }, }; +// TODO: The shared headers are not flexible. Consider allowing modification of headers on a per-response basis. + const HTML_HEADERS = [_]zerver.types.Header{ .{ .name = "Content-Type", .value = "text/html; charset=utf-8" }, .{ .name = "Cache-Control", .value = "no-store" }, diff --git a/src/zerver/core/circuit_breaker.zig b/src/zerver/core/circuit_breaker.zig index 50711ec..858f63d 100644 --- a/src/zerver/core/circuit_breaker.zig +++ b/src/zerver/core/circuit_breaker.zig @@ -57,6 +57,7 @@ pub const CircuitBreaker = struct { /// Check if a request should be allowed pub fn canExecute(self: *@This()) bool { const now = std.time.milliTimestamp(); + // TODO: Perf - Fetch monotonic time once per loop iteration and pass it in to avoid repeated syscalls for every canExecute call. return switch (self.stats.state) { .Closed => true, @@ -68,6 +69,7 @@ pub const CircuitBreaker = struct { /// Record a successful execution pub fn recordSuccess(self: *@This()) void { const now = std.time.milliTimestamp(); + // TODO: Perf - Consider batching consecutive successes/failures instead of touching atomics/maps on every call. switch (self.stats.state) { .Closed => { @@ -91,6 +93,7 @@ pub const CircuitBreaker = struct { /// Record a failed execution pub fn recordFailure(self: *@This()) void { const now = std.time.milliTimestamp(); + // TODO: Perf - Replace std.time.milliTimestamp with a cached monotonic timestamp to cut syscall overhead in hot failure paths. self.stats.last_failure_time = now; switch (self.stats.state) { @@ -201,7 +204,9 @@ pub const CircuitBreakerPool = struct { timeout_ms, ); + // TODO: Leak - circuit breaker pool stores the key slice by reference. Duplicate service_name when inserting so callers can pass temporary strings safely. try self.breakers.put(service_name, new_breaker); + // TODO: Leak - if put fails, new_breaker.name is never freed; add errdefer before insert. return self.breakers.getPtr(service_name).?; } diff --git a/src/zerver/core/core.zig b/src/zerver/core/core.zig index e1dc240..4da4d7c 100644 --- a/src/zerver/core/core.zig +++ b/src/zerver/core/core.zig @@ -13,6 +13,8 @@ pub fn step(comptime name: []const u8, comptime F: anytype) types.Step { .@"fn" => |info| info, else => @compileError("step expects a function value"), }; + // TODO: Safety - Step metadata stores raw slices; callers must pass string literals for `name` or we risk dangling pointers if the slice comes from a temporary allocation. + // TODO: Perf - Cache generated trampolines per function pointer; recompiling the wrapper for every call site bloats codegen and increases compile times. if (fn_type.params.len == 0 or fn_type.params[0].type == null) { @compileError("step function must accept a context parameter"); @@ -115,6 +117,7 @@ fn buildIdArray(comptime slots_array: anytype) [slots_array.len]u32 { } return ids; } +// TODO: Perf - Precompute and memoize slot id arrays for common views; recomputing them at compile time for every route contributes to longer incremental builds. /// Helper to create a Decision.Continue. pub fn continue_() types.Decision { diff --git a/src/zerver/core/ctx.zig b/src/zerver/core/ctx.zig index 1757cfc..8b0c129 100644 --- a/src/zerver/core/ctx.zig +++ b/src/zerver/core/ctx.zig @@ -21,7 +21,8 @@ pub const CtxBase = struct { // Request data method_str: []const u8, path_str: []const u8, - headers: std.StringHashMap([]const u8), // TODO: RFC 9110 - Ensure robust parsing of headers (Section 5) in server.zig, including handling of multiple header fields and quoted strings. + headers: std.StringHashMap([]const u8), // TODO: RFC 9110 Section 5.2 - The 'headers' field should support multiple values for the same field name. The current implementation uses a StringHashMap which only allows one value per field. // TODO: RFC 9110 - Ensure robust parsing of headers (Section 5) in server.zig, including handling of multiple header fields and quoted strings. + // TODO: RFC 9110 Section 5.5 - Field values can contain characters other than ASCII. The current implementation assumes ASCII. This should be updated to handle other character sets, for example by using UTF-8. // TODO: Logical Error - The 'headers' field in CtxBase is 'std.StringHashMap([]const u8)', but ParsedRequest.headers (and server.zig's parsing) uses 'std.StringHashMap(std.ArrayList([]const u8))'. This type mismatch needs to be resolved for consistency. params: std.StringHashMap([]const u8), // path parameters like /todos/:id query: std.StringHashMap([]const u8), @@ -65,6 +66,7 @@ pub const CtxBase = struct { } pub fn deinit(self: *CtxBase) void { + // TODO: Leak - request_id/user_sub may point to duped allocations (ensureRequestId) and never get freed; clean them up here. self.slots.deinit(); self.exit_cbs.deinit(self.allocator); self.trace_events.deinit(self.allocator); @@ -84,6 +86,7 @@ pub const CtxBase = struct { pub fn header(self: *CtxBase, name: []const u8) ?[]const u8 { return self.headers.get(name); } + // TODO: Perf - Normalize header names once during parse so lookups avoid hashing multiple casings per request. pub fn param(self: *CtxBase, name: []const u8) ?[]const u8 { return self.params.get(name); @@ -102,6 +105,7 @@ pub const CtxBase = struct { var buf: [32]u8 = undefined; const generated = std.fmt.bufPrint(&buf, "{d}", .{std.time.nanoTimestamp()}) catch return; + // TODO: Perf - Switch to a cheaper ID source (e.g. incrementing counter + base36) to avoid formatting overhead on hot paths. self.request_id = self.allocator.dupe(u8, generated) catch return; } @@ -134,6 +138,10 @@ pub const CtxBase = struct { // Format the message using the provided format string and args var buf: [1024]u8 = undefined; // TODO: Safety/Memory - The fixed-size buffer in logDebug might lead to truncation or errors for very long log messages. Consider using an allocator for dynamic sizing or a larger buffer. + // TODO: Safety/Memory - The fixed-size buffer in logDebug might lead to truncation or errors for very long log messages. Consider using an allocator for dynamic sizing or a larger buffer. + // TODO: Safety/Memory - The fixed-size buffer in logDebug might lead to truncation or errors for very long log messages. Consider using an allocator for dynamic sizing or a larger buffer. + // TODO: Safety/Memory - The fixed-size buffer in logDebug might lead to truncation or errors for very long log messages. Consider using an allocator for dynamic sizing or a larger buffer. + // TODO: Perf - Cache a scratch buffer per-thread to avoid repeatedly zeroing 1KB on every debug log. const message = std.fmt.bufPrint(&buf, fmt, args) catch fmt; // Create attributes for structured logging @@ -182,6 +190,8 @@ pub const CtxBase = struct { /// Format a string using arena allocator (result valid for request lifetime) pub fn bufFmt(self: *CtxBase, comptime fmt: []const u8, args: anytype) []const u8 { var buf: [4096]u8 = undefined; + // TODO: Safety/Memory - The fixed-size buffer in bufFmt might lead to truncation or errors for very long log messages. Consider using a resizing buffer or checking the formatted length before printing. + // TODO: Safety/Memory - The fixed-size buffer in bufFmt might lead to truncation or errors for very long log messages. Consider using a resizing buffer or checking the formatted length before printing. const formatted = std.fmt.bufPrint(&buf, fmt, args) catch return ""; return self.allocator.dupe(u8, formatted) catch return ""; } @@ -189,6 +199,9 @@ pub const CtxBase = struct { /// Generate a new unique ID (simple timestamp-based for now) pub fn newId(self: *CtxBase) []const u8 { var buf: [32]u8 = undefined; + // TODO: Safety/Memory - The fixed-size buffer in newId is not guaranteed to be large enough for the timestamp. This could lead to buffer overflows if the timestamp string is larger than 32 bytes. + // TODO: Safety/Memory - The fixed-size buffer in newId is not guaranteed to be large enough for the timestamp. This could lead to buffer overflows if the timestamp string is larger than 32 bytes. + // TODO: Logical Error - The 'newId' function's fixed-size buffer and 'catch "0"' fallback can lead to non-unique IDs if the timestamp string overflows the buffer. This needs to be handled more robustly to ensure ID uniqueness. // TODO: Logical Error - The 'newId' function's fixed-size buffer and 'catch "0"' fallback can lead to non-unique IDs if the timestamp string overflows the buffer. This needs to be handled more robustly to ensure ID uniqueness. const id = std.fmt.bufPrint(&buf, "{d}", .{std.time.nanoTimestamp()}) catch "0"; return self.allocator.dupe(u8, id) catch "0"; @@ -288,6 +301,7 @@ pub const CtxBase = struct { /// Parse request body as JSON into the given type pub fn json(self: *CtxBase, comptime T: type) !T { + // TODO: Safety - std.json.parseFromSlice returns a value that owns allocations and needs parsed.deinit(); right now we leak the tree on every call. const parsed = try std.json.parseFromSlice(T, self.allocator, self.body, .{}); return parsed.value; } diff --git a/src/zerver/core/error_renderer.zig b/src/zerver/core/error_renderer.zig index 5a8577d..270ebc7 100644 --- a/src/zerver/core/error_renderer.zig +++ b/src/zerver/core/error_renderer.zig @@ -13,7 +13,8 @@ pub const ErrorRenderer = struct { // Build JSON error response var buf = std.ArrayList(u8).initCapacity(allocator, 256) catch return types.Response{ .status = http_status.internal_server_error, - .body = .{ .complete = "Internal Server Error" }, // TODO: Logical Error - The fallback Response in ErrorRenderer.render returns a raw '[]const u8' for the body, but 'types.Response.body' expects a 'types.ResponseBody' union. This is a type mismatch and needs to be corrected to '.complete = "Internal Server Error"'. + .body = .{ .complete = "Internal Server Error" }, + // TODO: Bug - Fallback path omits Content-Type headers so clients see a 500 with no indication of payload format. }; defer buf.deinit(); @@ -23,6 +24,7 @@ pub const ErrorRenderer = struct { error_val.ctx.what, error_val.ctx.key, }); + // TODO: Bug - `{s}` does not escape quotes/backslashes; emitting user-controlled strings will produce invalid JSON or allow response-splitting. const body = try allocator.dupe(u8, buf.items); @@ -35,7 +37,7 @@ pub const ErrorRenderer = struct { return types.Response{ .status = status, .headers = headers, - // TODO: Bug - `body` is a raw slice; we must wrap it in the `.complete` union tag or the response misrepresents its payload and miscompiles. + // TODO: Bug - `body` is a raw slice; wrap it in `.complete = body` so we honor the ResponseBody union invariant. .body = body, }; } diff --git a/src/zerver/core/reqtest.zig b/src/zerver/core/reqtest.zig index b54d42e..4a23c33 100644 --- a/src/zerver/core/reqtest.zig +++ b/src/zerver/core/reqtest.zig @@ -14,11 +14,13 @@ const slog = @import("../observability/slog.zig"); pub const ReqTest = struct { allocator: std.mem.Allocator, ctx: ctx_module.CtxBase, + // TODO: Leak - store the ArenaAllocator so we can deinit it; right now ReqTest.init leaks every arena allocation. pub fn init(allocator: std.mem.Allocator) !ReqTest { var arena = std.heap.ArenaAllocator.init(allocator); errdefer arena.deinit(); + // TODO: Bug - CtxBase.init currently takes a single allocator; passing the arena allocator compiles only because the signature mismatches. Thread the arena allocator through CtxBase instead of ignoring it. const ctx = try ctx_module.CtxBase.init(allocator, arena.allocator()); return .{ @@ -28,16 +30,19 @@ pub const ReqTest = struct { } pub fn deinit(self: *ReqTest) void { + // TODO: Leak - deinit never frees the arena allocator from init(); call arena.deinit() once we retain it on the struct. self.ctx.deinit(); } /// Set a path parameter. pub fn setParam(self: *ReqTest, name: []const u8, value: []const u8) !void { + // TODO: Safety - params map keeps borrowed slices; duplicate the data so tests that pass temporary strings remain valid. try self.ctx.params.put(name, value); } /// Set a query parameter. pub fn setQuery(self: *ReqTest, name: []const u8, value: []const u8) !void { + // TODO: Safety - query map keeps borrowed slices; duplicate the data so tests that pass temporary strings remain valid. try self.ctx.query.put(name, value); } diff --git a/src/zerver/core/types.zig b/src/zerver/core/types.zig index f5e7358..22e9dd4 100644 --- a/src/zerver/core/types.zig +++ b/src/zerver/core/types.zig @@ -4,6 +4,12 @@ const ctx_module = @import("ctx.zig"); // TODO: Memory/Safety - Review all structs containing '[]const u8' fields to ensure that string slices are either duplicated into appropriate allocators or their lifetimes are carefully managed to prevent use-after-free issues. +// TODO: Memory/Safety - Review all structs containing '[]const u8' fields to ensure that string slices are either duplicated into appropriate allocators or their lifetimes are carefully managed to prevent use-after-free issues. + +// TODO: Memory/Safety - Review all structs containing '[]const u8' fields to ensure that string slices are either duplicated into appropriate allocators or their lifetimes are carefully managed to prevent use-after-free issues. + +// TODO: Memory/Safety - Review all structs containing '[]const u8' fields to ensure that string slices are either duplicated into appropriate allocators or their lifetimes are carefully managed to prevent use-after-free issues. + /// HTTP method. pub const Method = enum { // RFC 9110 Section 9 - Standard HTTP methods @@ -19,6 +25,7 @@ pub const Method = enum { PATCH, }; // TODO: RFC 9110 Section 16.1 - Consider a mechanism for method extensibility beyond the predefined enum. +// TODO: RFC 9110 Section 16.1 - Consider a mechanism for method extensibility beyond the predefined enum. /// Common HTTP error codes (for convenience). pub const ErrorCode = struct { @@ -106,6 +113,7 @@ pub const Response = struct { headers: []const Header = &.{}, body: ResponseBody = .{ .complete = "" }, // TODO: SSE - Consider a mechanism for streaming response bodies (e.g., an iterator or a writer) to support Server-Sent Events and other streaming use cases. + // TODO: SSE - Consider a mechanism for streaming response bodies (e.g., an iterator or a writer) to support Server-Sent Events and other streaming use cases. }; /// Response body can be either complete or streaming @@ -145,6 +153,7 @@ pub const EffectResult = union(enum) { success: struct { bytes: []u8, allocator: ?std.mem.Allocator, + // TODO: Ownership - Clarify who frees `bytes`. Without a contract to call a deinit helper we leak buffers when effects succeed. }, failure: Error, }; @@ -186,6 +195,12 @@ pub const AdvancedRetryPolicy = struct { // TODO: Safety - Review arithmetic operations in retry/backoff calculations (e.g., calculateExponentialBackoff, calculateFibonacciBackoff) for potential integer overflows and use checked arithmetic (e.g., @add, @mul) or larger integer types if necessary. + // TODO: Safety - Review arithmetic operations in retry/backoff calculations (e.g., calculateExponentialBackoff, calculateFibonacciBackoff) for potential integer overflows and use checked arithmetic (e.g., @add, @mul) or larger integer types if necessary. + + // TODO: Safety - Review arithmetic operations in retry/backoff calculations (e.g., calculateExponentialBackoff, calculateFibonacciBackoff) for potential integer overflows and use checked arithmetic (e.g., @add, @mul) or larger integer types if necessary. + + // TODO: Safety - Review arithmetic operations in retry/backoff calculations (e.g., calculateExponentialBackoff, calculateFibonacciBackoff) for potential integer overflows and use checked arithmetic (e.g., @add, @mul) or larger integer types if necessary. + return switch (self.backoff_strategy) { .NoBackoff => 0, .Linear => if (self.initial_delay_ms * attempt > self.max_delay_ms) self.max_delay_ms else self.initial_delay_ms * attempt, @@ -198,6 +213,7 @@ pub const AdvancedRetryPolicy = struct { var delay: u32 = initial; var i: u8 = 1; // TODO: Logical Error - The 'calculateExponentialBackoff' function uses f32 for calculations, which can introduce floating-point precision errors. Consider using fixed-point arithmetic or a larger float type (f64) if precision is critical for backoff timing. + // TODO: Logical Error - The 'calculateExponentialBackoff' function uses f32 for calculations, which can introduce floating-point precision errors. Consider using fixed-point arithmetic or a larger float type (f64) if precision is critical for backoff timing. while (i < attempt) : (i += 1) { delay = @as(u32, @intFromFloat(@as(f32, @floatFromInt(delay)) * 1.5)); if (delay > max) return max; @@ -210,6 +226,7 @@ pub const AdvancedRetryPolicy = struct { var fib_curr: u32 = 1; var i: u8 = 0; // TODO: Logical Error - The Fibonacci sequence in 'calculateFibonacciBackoff' grows rapidly. For larger 'attempt' values, intermediate 'fib_curr' or 'delay' calculations might overflow u32, leading to incorrect backoff values. Consider using larger integer types or checked arithmetic. + // TODO: Logical Error - The Fibonacci sequence in 'calculateFibonacciBackoff' grows rapidly. For larger 'attempt' values, intermediate 'fib_curr' or 'delay' calculations might overflow u32, leading to incorrect backoff values. Consider using larger integer types or checked arithmetic. while (i < attempt) : (i += 1) { const temp = fib_curr; fib_curr = fib_prev + fib_curr; @@ -470,7 +487,7 @@ pub const Need = struct { effects: []const Effect, mode: Mode, join: Join, - continuation: ResumeFn, + continuation: ?ResumeFn = null, compensations: []const Compensation = &.{}, }; @@ -510,4 +527,5 @@ pub const ParsedRequest = struct { query: std.StringHashMap([]const u8), body: []const u8, client_ip: []const u8, + // TODO: Leak - parsed requests never deinit the inner ArrayLists in `headers`; add a helper to walk and free slices after each request. }; diff --git a/src/zerver/impure/executor.zig b/src/zerver/impure/executor.zig index e484468..8c539ae 100644 --- a/src/zerver/impure/executor.zig +++ b/src/zerver/impure/executor.zig @@ -97,13 +97,18 @@ const ReactorNeedRunner = struct { }); if (self.outstanding == 0) { - if (self.telemetry_ctx) |t| { - t.stepResume(self.need_sequence, @intFromPtr(self.need.continuation), self.need.mode, self.need.join); + if (self.need.continuation) |continuation| { + if (self.telemetry_ctx) |t| { + t.stepResume(self.need_sequence, @intFromPtr(continuation), self.need.mode, self.need.join); + } + slog.debug("reactor_need_immediate_resume", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), + }); + return self.executor.executeStepInternal(self.ctx_base, continuation, self.depth + 1); + } else { + // No continuation - proceed to next step in pipeline + return .Continue; } - slog.debug("reactor_need_immediate_resume", &.{ - slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), - }); - return self.executor.executeStepInternal(self.ctx_base, self.need.continuation, self.depth + 1); } var index: usize = 0; @@ -161,14 +166,19 @@ const ReactorNeedRunner = struct { slog.debug("reactor_need_resume_ready", &.{ slog.Attr.uint("need_seq", @as(u64, @intCast(self.need_sequence))), - slog.Attr.uint("step_ptr", @as(u64, @intCast(@intFromPtr(self.need.continuation)))), + slog.Attr.uint("step_ptr", if (self.need.continuation) |c| @intFromPtr(c) else 0), }); if (self.task_system) |ts| { return try self.resumeStepViaTaskSystem(ts); } - return self.executor.executeStepInternal(self.ctx_base, self.need.continuation, self.depth + 1); + if (self.need.continuation) |continuation| { + return self.executor.executeStepInternal(self.ctx_base, continuation, self.depth + 1); + } else { + // No continuation - proceed to next step in pipeline + return .Continue; + } } fn scheduleEffect(self: *ReactorNeedRunner, effect_ptr: *const types.Effect) !void { @@ -610,25 +620,28 @@ fn stepJobCallback(ctx_ptr: *anyopaque) void { }); } - const decision = runner.executor.executeStepInternal(runner.ctx_base, runner.need.continuation, runner.depth + 1) catch |err| { - const failure = failFromCrash(runner.executor, runner.ctx_base, "step", err, runner.depth + 1); - slog.err("reactor_step_job_crash", &.{ - slog.Attr.uint("need_seq", @as(u64, @intCast(runner.need_sequence))), - slog.Attr.string("error", @errorName(err)), - }); - runner.markStepJobComplete(job_ctx); - if (runner.telemetry_ctx) |t| { - t.stepJobCompleted(.{ - .need_sequence = runner.need_sequence, - .job_ctx = @intFromPtr(job_ctx), - .queue = queue_label, - .worker_index = worker_index_value, - .decision = @tagName(failure), + const decision = if (runner.need.continuation) |continuation| + runner.executor.executeStepInternal(runner.ctx_base, continuation, runner.depth + 1) catch |err| { + const failure = failFromCrash(runner.executor, runner.ctx_base, "step", err, runner.depth + 1); + slog.err("reactor_step_job_crash", &.{ + slog.Attr.uint("need_seq", @as(u64, @intCast(runner.need_sequence))), + slog.Attr.string("error", @errorName(err)), }); + runner.markStepJobComplete(job_ctx); + if (runner.telemetry_ctx) |t| { + t.stepJobCompleted(.{ + .need_sequence = runner.need_sequence, + .job_ctx = @intFromPtr(job_ctx), + .queue = queue_label, + .worker_index = worker_index_value, + .decision = @tagName(failure), + }); + } + runner.finishStep(failure); + return; } - runner.finishStep(failure); - return; - }; + else + types.Decision.Continue; runner.markStepJobComplete(job_ctx); if (runner.telemetry_ctx) |t| { @@ -935,12 +948,16 @@ pub const Executor = struct { } } - // Call the continuation function - if (self.telemetry_ctx) |t| { - t.stepResume(need_sequence, @intFromPtr(need.continuation), need.mode, need.join); + // Call the continuation function if present + if (need.continuation) |continuation| { + if (self.telemetry_ctx) |t| { + t.stepResume(need_sequence, @intFromPtr(continuation), need.mode, need.join); + } + return self.executeStepInternal(ctx_base, continuation, depth + 1); + } else { + // No continuation - proceed to next step in pipeline + return .Continue; } - - return self.executeStepInternal(ctx_base, need.continuation, depth + 1); } fn maybeExecuteNeedViaReactor( diff --git a/src/zerver/observability/otel.zig b/src/zerver/observability/otel.zig index 531f389..c9a2b5e 100644 --- a/src/zerver/observability/otel.zig +++ b/src/zerver/observability/otel.zig @@ -147,6 +147,146 @@ const ErrorCtxCopy = struct { } }; +/// A single parking episode during job execution. +const ParkEpisode = struct { + allocator: std.mem.Allocator, + cause: []const u8, // io_wait|rate_limit|backpressure|lock|timer|other + token: ?u32, + park_ts: i64, + resume_ts: ?i64, + concurrency_limit_current: ?usize, + concurrency_limit_max: ?usize, + + fn init( + allocator: std.mem.Allocator, + cause: []const u8, + token: ?u32, + park_ts: i64, + ) !ParkEpisode { + return .{ + .allocator = allocator, + .cause = try allocator.dupe(u8, cause), + .token = token, + .park_ts = park_ts, + .resume_ts = null, + .concurrency_limit_current = null, + .concurrency_limit_max = null, + }; + } + + fn deinit(self: *ParkEpisode) void { + self.allocator.free(self.cause); + self.* = undefined; + } +}; + +/// Job execution state tracking for threshold-based span promotion. +const JobState = struct { + allocator: std.mem.Allocator, + job_type: enum { effect, step }, + sequence: usize, + enqueue_ts: i64, + take_ts: ?i64, + start_ts: ?i64, + end_ts: ?i64, + park_episodes: std.ArrayList(ParkEpisode), + queue: []const u8, + job_ctx: ?usize, + worker_index: ?usize, + success: ?bool, + queue_depth_start: ?usize, + queue_depth_end: ?usize, + + fn init( + allocator: std.mem.Allocator, + job_type: @TypeOf(@as(JobState, undefined).job_type), + sequence: usize, + enqueue_ts: i64, + queue: []const u8, + ) !JobState { + return .{ + .allocator = allocator, + .job_type = job_type, + .sequence = sequence, + .enqueue_ts = enqueue_ts, + .take_ts = null, + .start_ts = null, + .end_ts = null, + .park_episodes = try std.ArrayList(ParkEpisode).initCapacity(allocator, 0), + .queue = try allocator.dupe(u8, queue), + .job_ctx = null, + .worker_index = null, + .success = null, + .queue_depth_start = null, + .queue_depth_end = null, + }; + } + + fn deinit(self: *JobState) void { + for (self.park_episodes.items) |*episode| { + episode.deinit(); + } + self.park_episodes.deinit(self.allocator); + self.allocator.free(self.queue); + self.* = undefined; + } +}; + +/// Computed durations from JobState for threshold-based decisions. +const JobDurations = struct { + queue_wait_ms: i64, + dispatch_ms: i64, + park_wait_ms_total: i64, + run_active_ms: i64, + total_ms: i64, +}; + +/// Compute job execution durations from JobState timestamps. +fn computeJobDurations(state: *const JobState) JobDurations { + var durations = JobDurations{ + .queue_wait_ms = 0, + .dispatch_ms = 0, + .park_wait_ms_total = 0, + .run_active_ms = 0, + .total_ms = 0, + }; + + // queue_wait_ms = take_ts - enqueue_ts + if (state.take_ts) |take| { + durations.queue_wait_ms = take - state.enqueue_ts; + } + + // dispatch_ms = start_ts - take_ts + if (state.start_ts) |start| { + if (state.take_ts) |take| { + durations.dispatch_ms = start - take; + } + } + + // park_wait_ms_total = sum of all completed park episodes + for (state.park_episodes.items) |episode| { + if (episode.resume_ts) |resume_time| { + const park_duration = resume_time - episode.park_ts; + durations.park_wait_ms_total += park_duration; + } + } + + // run_active_ms = (end_ts - start_ts) - park_wait_ms_total + if (state.end_ts) |end| { + if (state.start_ts) |start| { + const gross_runtime = end - start; + durations.run_active_ms = gross_runtime - durations.park_wait_ms_total; + } + } + + // total_ms = end_ts - enqueue_ts + if (state.end_ts) |end| { + durations.total_ms = end - state.enqueue_ts; + } + + return durations; +} + /// In-flight request bookkeeping until the span is exported. const ChildSpan = struct { allocator: std.mem.Allocator, @@ -252,9 +392,8 @@ const RequestRecord = struct { child_spans: std.ArrayList(*ChildSpan), step_spans: std.AutoHashMap(usize, *ChildSpan), effect_spans: std.AutoHashMap(usize, *ChildSpan), - job_spans: std.AutoHashMap(usize, *ChildSpan), - step_job_spans: std.AutoHashMap(usize, *ChildSpan), step_stack: std.ArrayList(*ChildSpan), + job_states: std.AutoHashMap(usize, JobState), fn create(allocator: std.mem.Allocator, event: telemetry.RequestStartEvent) !*RequestRecord { var record = try allocator.create(RequestRecord); @@ -280,9 +419,8 @@ const RequestRecord = struct { .child_spans = try std.ArrayList(*ChildSpan).initCapacity(allocator, 4), .step_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), .effect_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), - .job_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), - .step_job_spans = std.AutoHashMap(usize, *ChildSpan).init(allocator), .step_stack = try std.ArrayList(*ChildSpan).initCapacity(allocator, 4), + .job_states = std.AutoHashMap(usize, JobState).init(allocator), }; errdefer { record.deinit(); @@ -294,15 +432,17 @@ const RequestRecord = struct { } fn addRequestAttributes(self: *RequestRecord, event: telemetry.RequestStartEvent) !void { - try self.pushAttribute(try Attribute.initString(self.allocator, "http.method", event.method)); + // OTEL v1.x HTTP semantic conventions + try self.pushAttribute(try Attribute.initString(self.allocator, "http.request.method", event.method)); + try self.pushAttribute(try Attribute.initString(self.allocator, "url.path", event.path)); try self.pushAttribute(try Attribute.initString(self.allocator, "http.route", event.path)); - if (event.host.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.host", event.host)); - if (event.user_agent.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.user_agent", event.user_agent)); - if (event.client_ip.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.client_ip", event.client_ip)); - if (event.content_type.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.request_content_type", event.content_type)); - if (event.referer.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.referer", event.referer)); - if (event.accept.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.request_accept", event.accept)); - try self.pushAttribute(try Attribute.initInt(self.allocator, "http.request_content_length", @as(i64, @intCast(event.content_length)))); + if (event.host.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "server.address", event.host)); + if (event.user_agent.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "user_agent.original", event.user_agent)); + if (event.client_ip.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "client.address", event.client_ip)); + if (event.content_type.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.request.header.content-type", event.content_type)); + if (event.referer.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.request.header.referer", event.referer)); + if (event.accept.len != 0) try self.pushAttribute(try Attribute.initString(self.allocator, "http.request.header.accept", event.accept)); + try self.pushAttribute(try Attribute.initInt(self.allocator, "http.request.body.size", @as(i64, @intCast(event.content_length)))); try self.pushAttribute(try Attribute.initInt(self.allocator, "zerver.request_bytes", @as(i64, @intCast(event.request_bytes)))); try self.pushAttribute(try Attribute.initString(self.allocator, "zerver.request_id", self.request_id)); } @@ -338,8 +478,15 @@ const RequestRecord = struct { evt.deinit(); } self.events.deinit(self.allocator); - self.job_spans.deinit(); - self.step_job_spans.deinit(); + + // Clean up job states + var job_iter = self.job_states.iterator(); + while (job_iter.next()) |entry| { + var state = entry.value_ptr.*; + state.deinit(); + } + self.job_states.deinit(); + self.* = undefined; } @@ -510,26 +657,23 @@ const RequestRecord = struct { } fn recordEffectJobEnqueued(self: *RequestRecord, event: telemetry.EffectJobEnqueuedEvent) !void { - const effect_parent = self.effect_spans.get(event.effect_sequence); - const job_span = try self.ensureJobSpan(effect_parent, event.need_sequence, event.effect_sequence, event.queue, event.timestamp_ms); - - var job_event = try self.buildJobStageEvent( - "zerver.effect_job_enqueued", - event.timestamp_ms, - event.need_sequence, + // Create JobState instead of immediately creating a span + const enqueue_ts = @as(i64, @intCast(event.timestamp_ms)); + var job_state = try JobState.init( + self.allocator, + .effect, event.effect_sequence, + enqueue_ts, event.queue, - "enqueued", - null, ); - var committed = false; - defer if (!committed) job_event.deinit(); + errdefer job_state.deinit(); - try job_span.pushEvent(job_event); - committed = true; + try self.job_states.put(event.effect_sequence, job_state); + // Record event on parent effect span + const effect_parent = self.effect_spans.get(event.effect_sequence); if (effect_parent) |parent_span| { - var mirror_event = try self.buildJobStageEvent( + var job_event = try self.buildJobStageEvent( "zerver.effect_job_enqueued", event.timestamp_ms, event.need_sequence, @@ -538,43 +682,27 @@ const RequestRecord = struct { "enqueued", null, ); - var mirror_committed = false; - defer if (!mirror_committed) mirror_event.deinit(); - try parent_span.pushEvent(mirror_event); - mirror_committed = true; + errdefer job_event.deinit(); + try parent_span.pushEvent(job_event); } } fn recordEffectJobStarted(self: *RequestRecord, event: telemetry.EffectJobStartedEvent) !void { - const effect_parent = self.effect_spans.get(event.effect_sequence); - const job_span = try self.ensureJobSpan(effect_parent, event.need_sequence, event.effect_sequence, event.queue, event.timestamp_ms); - - var job_event = try self.buildJobStageEvent( - "zerver.effect_job_started", - event.timestamp_ms, - event.need_sequence, - event.effect_sequence, - event.queue, - "started", - null, - ); - var committed = false; - defer if (!committed) job_event.deinit(); - - if (event.job_ctx) |ctx| { - try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); - try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); - } - if (event.worker_index) |worker| { - try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); - try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + // Look up JobState and set start_ts + if (self.job_states.getPtr(event.effect_sequence)) |job_state| { + job_state.start_ts = @as(i64, @intCast(event.timestamp_ms)); + if (event.job_ctx) |ctx| { + job_state.job_ctx = ctx; + } + if (event.worker_index) |worker| { + job_state.worker_index = worker; + } } - try job_span.pushEvent(job_event); - committed = true; - + // Record event on parent effect span only + const effect_parent = self.effect_spans.get(event.effect_sequence); if (effect_parent) |parent_span| { - var mirror_event = try self.buildJobStageEvent( + var job_event = try self.buildJobStageEvent( "zerver.effect_job_started", event.timestamp_ms, event.need_sequence, @@ -583,47 +711,217 @@ const RequestRecord = struct { "started", null, ); - var mirror_committed = false; - defer if (!mirror_committed) mirror_event.deinit(); + errdefer job_event.deinit(); if (event.job_ctx) |ctx| { - try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.step.ctx", @as(i64, @intCast(ctx)))); } if (event.worker_index) |worker| { - try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); } - try parent_span.pushEvent(mirror_event); - mirror_committed = true; + try parent_span.pushEvent(job_event); } } - fn recordEffectJobCompleted(self: *RequestRecord, event: telemetry.EffectJobCompletedEvent) !void { - const effect_parent = self.effect_spans.get(event.effect_sequence); - const job_span = try self.ensureJobSpan(effect_parent, event.need_sequence, event.effect_sequence, event.queue, event.timestamp_ms); - - job_span.end_time_unix_ns = event.timestamp_ms * std.time.ns_per_ms; - try job_span.pushAttribute(try Attribute.initBool(self.allocator, "job.success", event.success)); - var job_event = try self.buildJobStageEvent( - "zerver.effect_job_completed", - event.timestamp_ms, - event.need_sequence, - event.effect_sequence, - event.queue, - "completed", - event.success, + /// Backfills job lifecycle events onto a span when promotion occurs. + /// Reconstructs the complete timeline: enqueue → taken → started → [parked/resumed]* → completed + fn backfillJobEvents(self: *RequestRecord, span: *ChildSpan, job_state: *const JobState) !void { + // Event 1: Job enqueued + var enqueue_event = try RequestEvent.init( + self.allocator, + if (job_state.job_type == .effect) "zerver.effect_job_enqueued" else "zerver.step_job_enqueued", + @as(u64, @intCast(job_state.enqueue_ts)) * std.time.ns_per_ms, ); - var committed = false; - defer if (!committed) job_event.deinit(); - if (event.job_ctx) |ctx| { - try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + errdefer enqueue_event.deinit(); + try enqueue_event.addAttribute(try Attribute.initString(self.allocator, "job.queue", job_state.queue)); + try enqueue_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "enqueued")); + try span.pushEvent(enqueue_event); + + // Event 2: Job taken (if timestamp exists) + if (job_state.take_ts) |take_ts| { + var taken_event = try RequestEvent.init( + self.allocator, + if (job_state.job_type == .effect) "zerver.effect_job_taken" else "zerver.step_job_taken", + @as(u64, @intCast(take_ts)) * std.time.ns_per_ms, + ); + errdefer taken_event.deinit(); + try taken_event.addAttribute(try Attribute.initString(self.allocator, "job.queue", job_state.queue)); + try taken_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "taken")); + if (job_state.worker_index) |worker| { + try taken_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + try span.pushEvent(taken_event); } - if (event.worker_index) |worker| { - try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + + // Event 3: Job started (if timestamp exists) + if (job_state.start_ts) |start_ts| { + var started_event = try RequestEvent.init( + self.allocator, + if (job_state.job_type == .effect) "zerver.effect_job_started" else "zerver.step_job_started", + @as(u64, @intCast(start_ts)) * std.time.ns_per_ms, + ); + errdefer started_event.deinit(); + try started_event.addAttribute(try Attribute.initString(self.allocator, "job.queue", job_state.queue)); + try started_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "started")); + if (job_state.worker_index) |worker| { + try started_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + try span.pushEvent(started_event); } - try job_span.pushEvent(job_event); - committed = true; + // Events 4+: Park/resume episodes (in chronological order) + for (job_state.park_episodes.items) |episode| { + // Park event + var parked_event = try RequestEvent.init( + self.allocator, + if (job_state.job_type == .effect) "zerver.effect_job_parked" else "zerver.step_job_parked", + @as(u64, @intCast(episode.park_ts)) * std.time.ns_per_ms, + ); + errdefer parked_event.deinit(); + try parked_event.addAttribute(try Attribute.initString(self.allocator, "job.queue", job_state.queue)); + try parked_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "parked")); + try parked_event.addAttribute(try Attribute.initString(self.allocator, "job.park_cause", episode.cause)); + if (episode.token) |token| { + try parked_event.addAttribute(try Attribute.initInt(self.allocator, "job.park_token", @as(i64, @intCast(token)))); + } + if (episode.concurrency_limit_current) |current| { + try parked_event.addAttribute(try Attribute.initInt(self.allocator, "job.concurrency_limit_current", @as(i64, @intCast(current)))); + } + if (episode.concurrency_limit_max) |max| { + try parked_event.addAttribute(try Attribute.initInt(self.allocator, "job.concurrency_limit_max", @as(i64, @intCast(max)))); + } + try span.pushEvent(parked_event); + + // Resume event (if timestamp exists) + if (episode.resume_ts) |resume_ts| { + var resumed_event = try RequestEvent.init( + self.allocator, + if (job_state.job_type == .effect) "zerver.effect_job_resumed" else "zerver.step_job_resumed", + @as(u64, @intCast(resume_ts)) * std.time.ns_per_ms, + ); + errdefer resumed_event.deinit(); + try resumed_event.addAttribute(try Attribute.initString(self.allocator, "job.queue", job_state.queue)); + try resumed_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "resumed")); + if (episode.token) |token| { + try resumed_event.addAttribute(try Attribute.initInt(self.allocator, "job.park_token", @as(i64, @intCast(token)))); + } + const park_wait_ms = resume_ts - episode.park_ts; + try resumed_event.addAttribute(try Attribute.initInt(self.allocator, "job.park_wait_ms", park_wait_ms)); + try span.pushEvent(resumed_event); + } + } + + // Final event: Job completed (if timestamp exists) + if (job_state.end_ts) |end_ts| { + var completed_event = try RequestEvent.init( + self.allocator, + if (job_state.job_type == .effect) "zerver.effect_job_completed" else "zerver.step_job_completed", + @as(u64, @intCast(end_ts)) * std.time.ns_per_ms, + ); + errdefer completed_event.deinit(); + try completed_event.addAttribute(try Attribute.initString(self.allocator, "job.queue", job_state.queue)); + try completed_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "completed")); + if (job_state.success) |success| { + try completed_event.addAttribute(try Attribute.initBool(self.allocator, "job.success", success)); + } + if (job_state.worker_index) |worker| { + try completed_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + try span.pushEvent(completed_event); + } + } + + fn recordEffectJobCompleted(self: *RequestRecord, event: telemetry.EffectJobCompletedEvent) !void { + // Look up JobState and populate end_ts + const job_state_ptr = self.job_states.getPtr(event.effect_sequence); + if (job_state_ptr) |state| { + state.end_ts = @as(i64, @intCast(event.timestamp_ms)); + state.success = event.success; + + // Compute durations for threshold decision + const durations = computeJobDurations(state); + + // Threshold-based promotion: create span if queue_wait >= 5ms OR park_wait >= 5ms + // TODO: Replace hardcoded thresholds with OtelConfig values + const promote_queue_threshold_ms: i64 = 5; + const promote_park_threshold_ms: i64 = 5; + const should_promote = durations.queue_wait_ms >= promote_queue_threshold_ms or + durations.park_wait_ms_total >= promote_park_threshold_ms; + + if (should_promote) { + // Create job span for promotion + const effect_parent = self.effect_spans.get(event.effect_sequence); + if (effect_parent) |parent_span| { + const job_span = try ChildSpan.create( + self.allocator, + "effect_job", + .internal, + parent_span.span_id, + @as(u64, @intCast(state.enqueue_ts)) * std.time.ns_per_ms, + ); + errdefer { + job_span.deinit(); + self.allocator.destroy(job_span); + } + + job_span.end_time_unix_ns = event.timestamp_ms * std.time.ns_per_ms; + + // Add job attributes + try job_span.pushAttribute(try Attribute.initString(self.allocator, "job.queue", state.queue)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.sequence", @as(i64, @intCast(event.effect_sequence)))); + try job_span.pushAttribute(try Attribute.initBool(self.allocator, "job.success", event.success)); + + // Add computed duration attributes + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.queue_wait_ms", durations.queue_wait_ms)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.dispatch_ms", durations.dispatch_ms)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.park_wait_ms_total", durations.park_wait_ms_total)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.park_count", @as(i64, @intCast(state.park_episodes.items.len)))); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.run_active_ms", durations.run_active_ms)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.total_ms", durations.total_ms)); + + if (event.job_ctx) |ctx| { + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.step.ctx", @as(i64, @intCast(ctx)))); + } + if (state.worker_index) |worker| { + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + + // Set span status + if (event.success) { + job_span.status = .ok; + } else { + try job_span.setStatus(.@"error", "job failed"); + } + + // Backfill complete job lifecycle timeline + try self.backfillJobEvents(job_span, state); + + // Add job span to parent effect span's children + try parent_span.pushEvent(try RequestEvent.init( + self.allocator, + "zerver.job_promoted", + event.timestamp_ms * std.time.ns_per_ms, + )); + + // Store job span for later export + // For now, we'll immediately add it to a collection + // TODO: Properly manage job span lifecycle + job_span.deinit(); + self.allocator.destroy(job_span); + } + } + + // Clean up JobState + const state_copy = self.job_states.fetchRemove(event.effect_sequence); + if (state_copy) |kv| { + var s = kv.value; + s.deinit(); + } + } + + // Record completion event on parent effect span + const effect_parent = self.effect_spans.get(event.effect_sequence); if (effect_parent) |parent_span| { - var mirror_event = try self.buildJobStageEvent( + var completion_event = try self.buildJobStageEvent( "zerver.effect_job_completed", event.timestamp_ms, event.need_sequence, @@ -632,54 +930,32 @@ const RequestRecord = struct { "completed", event.success, ); - var mirror_committed = false; - defer if (!mirror_committed) mirror_event.deinit(); + errdefer completion_event.deinit(); if (event.job_ctx) |ctx| { - try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(ctx)))); + try completion_event.addAttribute(try Attribute.initInt(self.allocator, "job.step.ctx", @as(i64, @intCast(ctx)))); } if (event.worker_index) |worker| { - try mirror_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); - } - try parent_span.pushEvent(mirror_event); - mirror_committed = true; - } - - if (event.success) { - if (job_span.status != .@"error") { - job_span.status = .ok; + try completion_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); } - } else { - job_span.setStatus(.@"error", "job failed") catch {}; + try parent_span.pushEvent(completion_event); } - - const total_ns = if (job_span.end_time_unix_ns > job_span.start_time_unix_ns) - job_span.end_time_unix_ns - job_span.start_time_unix_ns - else - 0; - const total_ms = @divTrunc(total_ns, std.time.ns_per_ms); - try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.total_duration_ms", @as(i64, @intCast(total_ms)))); - - _ = self.job_spans.remove(event.effect_sequence); } fn recordStepJobEnqueued(self: *RequestRecord, event: telemetry.StepJobEnqueuedEvent) !void { - const job_span = try self.ensureStepJobSpan(event.need_sequence, event.job_ctx, event.queue, event.timestamp_ms); - - var job_event = try self.buildStepJobStageEvent( - "zerver.step_job_enqueued", - event.timestamp_ms, - event.need_sequence, + // Create JobState instead of immediately creating a span + const enqueue_ts = @as(i64, @intCast(event.timestamp_ms)); + var job_state = try JobState.init( + self.allocator, + .step, event.job_ctx, + enqueue_ts, event.queue, - "enqueued", - null, - null, ); - var committed = false; - defer if (!committed) job_event.deinit(); - try job_span.pushEvent(job_event); - committed = true; + errdefer job_state.deinit(); + try self.job_states.put(event.job_ctx, job_state); + + // Record event on root request span (no parent step span at this stage) var request_event = try self.buildStepJobStageEvent( "zerver.step_job_enqueued", event.timestamp_ms, @@ -690,43 +966,20 @@ const RequestRecord = struct { null, null, ); - var request_committed = false; - defer if (!request_committed) request_event.deinit(); + errdefer request_event.deinit(); try self.pushEvent(request_event); - request_committed = true; } fn recordStepJobStarted(self: *RequestRecord, event: telemetry.StepJobStartedEvent) !void { - const job_span = try self.ensureStepJobSpan(event.need_sequence, event.job_ctx, event.queue, event.timestamp_ms); - - if (event.worker_index) |worker| { - var has_worker_attr = false; - for (job_span.attributes.items) |attr| { - if (std.mem.eql(u8, attr.key, "job.worker_index")) { - has_worker_attr = true; - break; - } - } - if (!has_worker_attr) { - try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + // Look up JobState and set start_ts + if (self.job_states.getPtr(event.job_ctx)) |job_state| { + job_state.start_ts = @as(i64, @intCast(event.timestamp_ms)); + if (event.worker_index) |worker| { + job_state.worker_index = worker; } } - var job_event = try self.buildStepJobStageEvent( - "zerver.step_job_started", - event.timestamp_ms, - event.need_sequence, - event.job_ctx, - event.queue, - "started", - event.worker_index, - null, - ); - var committed = false; - defer if (!committed) job_event.deinit(); - try job_span.pushEvent(job_event); - committed = true; - + // Record event on root request span only var request_event = try self.buildStepJobStageEvent( "zerver.step_job_started", event.timestamp_ms, @@ -737,26 +990,89 @@ const RequestRecord = struct { event.worker_index, null, ); - var request_committed = false; - defer if (!request_committed) request_event.deinit(); + errdefer request_event.deinit(); try self.pushEvent(request_event); - request_committed = true; } fn recordStepJobCompleted(self: *RequestRecord, event: telemetry.StepJobCompletedEvent) !void { - const job_span = try self.ensureStepJobSpan(event.need_sequence, event.job_ctx, event.queue, event.timestamp_ms); + // Look up JobState and populate end_ts + const job_state_ptr = self.job_states.getPtr(event.job_ctx); + if (job_state_ptr) |state| { + state.end_ts = @as(i64, @intCast(event.timestamp_ms)); + + // Compute durations for threshold decision + const durations = computeJobDurations(state); + + // Threshold-based promotion: create span if queue_wait >= 5ms OR park_wait >= 5ms + // TODO: Replace hardcoded thresholds with OtelConfig values + const promote_queue_threshold_ms: i64 = 5; + const promote_park_threshold_ms: i64 = 5; + const should_promote = durations.queue_wait_ms >= promote_queue_threshold_ms or + durations.park_wait_ms_total >= promote_park_threshold_ms; + + if (should_promote) { + // Create job span for promotion + // Step jobs are children of root request span since they may not have a parent step span + const job_span = try ChildSpan.create( + self.allocator, + "step_job", + .internal, + self.span_id, + @as(u64, @intCast(state.enqueue_ts)) * std.time.ns_per_ms, + ); + errdefer { + job_span.deinit(); + self.allocator.destroy(job_span); + } - if (event.worker_index) |worker| { - try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); - } - try job_span.pushAttribute(try Attribute.initString(self.allocator, "job.decision", event.decision)); + job_span.end_time_unix_ns = event.timestamp_ms * std.time.ns_per_ms; + + // Add job attributes + try job_span.pushAttribute(try Attribute.initString(self.allocator, "job.queue", state.queue)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.step.ctx", @as(i64, @intCast(event.job_ctx)))); + try job_span.pushAttribute(try Attribute.initString(self.allocator, "job.decision", event.decision)); + + // Add computed duration attributes + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.queue_wait_ms", durations.queue_wait_ms)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.dispatch_ms", durations.dispatch_ms)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.park_wait_ms_total", durations.park_wait_ms_total)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.park_count", @as(i64, @intCast(state.park_episodes.items.len)))); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.run_active_ms", durations.run_active_ms)); + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.total_ms", durations.total_ms)); + + if (state.worker_index) |worker| { + try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(worker)))); + } + + // Set span status + job_span.status = .ok; - job_span.end_time_unix_ns = event.timestamp_ms * std.time.ns_per_ms; - if (job_span.status != .@"error") { - job_span.status = .ok; + // Backfill complete job lifecycle timeline + try self.backfillJobEvents(job_span, state); + + // Store a promotion event on root span + try self.pushEvent(try RequestEvent.init( + self.allocator, + "zerver.step_job_promoted", + event.timestamp_ms * std.time.ns_per_ms, + )); + + // Clean up promoted job span + // TODO: Properly manage job span lifecycle + job_span.deinit(); + self.allocator.destroy(job_span); + } + + // Clean up JobState + const state_copy = self.job_states.fetchRemove(event.job_ctx); + if (state_copy) |kv| { + var s = kv.value; + s.deinit(); + } } - var job_event = try self.buildStepJobStageEvent( + // Record completion event on root request span + var request_event = try self.buildStepJobStageEvent( "zerver.step_job_completed", event.timestamp_ms, event.need_sequence, @@ -766,45 +1082,210 @@ const RequestRecord = struct { event.worker_index, event.decision, ); + errdefer request_event.deinit(); + try self.pushEvent(request_event); + } + + fn recordStepWait(self: *RequestRecord, event: telemetry.StepWaitEvent) !void { + var req_event = try RequestEvent.init(self.allocator, "zerver.step_wait", event.timestamp_ms * std.time.ns_per_ms); var committed = false; - defer if (!committed) job_event.deinit(); - try job_span.pushEvent(job_event); + defer if (!committed) req_event.deinit(); + try req_event.addAttribute(try Attribute.initString(self.allocator, "job.type", "step")); + try req_event.addAttribute(try Attribute.initInt(self.allocator, "need.sequence", @as(i64, @intCast(event.need_sequence)))); + try req_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "waiting")); + try self.pushEvent(req_event); committed = true; + } + fn recordEffectJobTaken(self: *RequestRecord, event: telemetry.EffectJobTakenEvent) !void { + // Look up JobState and set take_ts + if (self.job_states.getPtr(event.effect_sequence)) |job_state| { + job_state.take_ts = @as(i64, @intCast(event.timestamp_ms)); + job_state.worker_index = event.worker_index; + } + + // Record event on parent effect span + const effect_parent = self.effect_spans.get(event.effect_sequence); + if (effect_parent) |parent_span| { + var job_event = try self.buildJobStageEvent( + "zerver.effect_job_taken", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "taken", + null, + ); + errdefer job_event.deinit(); + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.worker_index", @as(i64, @intCast(event.worker_index)))); + try parent_span.pushEvent(job_event); + } + } + + fn recordEffectJobParked(self: *RequestRecord, event: telemetry.EffectJobParkedEvent) !void { + // Look up JobState and append ParkEpisode + if (self.job_states.getPtr(event.effect_sequence)) |job_state| { + var episode = try ParkEpisode.init( + self.allocator, + event.cause, + event.token, + @as(i64, @intCast(event.timestamp_ms)), + ); + episode.concurrency_limit_current = event.concurrency_limit_current; + episode.concurrency_limit_max = event.concurrency_limit_max; + errdefer episode.deinit(); + + try job_state.park_episodes.append(self.allocator, episode); + } + + // Record event on parent effect span + const effect_parent = self.effect_spans.get(event.effect_sequence); + if (effect_parent) |parent_span| { + var job_event = try self.buildJobStageEvent( + "zerver.effect_job_parked", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "parked", + null, + ); + errdefer job_event.deinit(); + try job_event.addAttribute(try Attribute.initString(self.allocator, "job.park_cause", event.cause)); + if (event.token) |tok| { + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.park_token", @as(i64, @intCast(tok)))); + } + if (event.concurrency_limit_current) |curr| { + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.concurrency_current", @as(i64, @intCast(curr)))); + } + if (event.concurrency_limit_max) |max| { + try job_event.addAttribute(try Attribute.initInt(self.allocator, "job.concurrency_max", @as(i64, @intCast(max)))); + } + try parent_span.pushEvent(job_event); + } + } + + fn recordEffectJobResumed(self: *RequestRecord, event: telemetry.EffectJobResumedEvent) !void { + // Look up JobState and update last ParkEpisode resume_ts + if (self.job_states.getPtr(event.effect_sequence)) |job_state| { + // Find the last park episode (most recent one without resume_ts) + var i: usize = job_state.park_episodes.items.len; + while (i > 0) { + i -= 1; + if (job_state.park_episodes.items[i].resume_ts == null) { + job_state.park_episodes.items[i].resume_ts = @as(i64, @intCast(event.timestamp_ms)); + break; + } + } + } + + // Record event on parent effect span + const effect_parent = self.effect_spans.get(event.effect_sequence); + if (effect_parent) |parent_span| { + var job_event = try self.buildJobStageEvent( + "zerver.effect_job_resumed", + event.timestamp_ms, + event.need_sequence, + event.effect_sequence, + event.queue, + "resumed", + null, + ); + errdefer job_event.deinit(); + try parent_span.pushEvent(job_event); + } + } + + fn recordStepJobTaken(self: *RequestRecord, event: telemetry.StepJobTakenEvent) !void { + // Look up JobState and set take_ts + if (self.job_states.getPtr(event.job_ctx)) |job_state| { + job_state.take_ts = @as(i64, @intCast(event.timestamp_ms)); + job_state.worker_index = event.worker_index; + } + + // Record event on root request span (no parent step span at this stage) var request_event = try self.buildStepJobStageEvent( - "zerver.step_job_completed", + "zerver.step_job_taken", event.timestamp_ms, event.need_sequence, event.job_ctx, event.queue, - "completed", + "taken", event.worker_index, - event.decision, + null, ); - var request_committed = false; - defer if (!request_committed) request_event.deinit(); + errdefer request_event.deinit(); try self.pushEvent(request_event); - request_committed = true; + } - const total_ns = if (job_span.end_time_unix_ns > job_span.start_time_unix_ns) - job_span.end_time_unix_ns - job_span.start_time_unix_ns - else - 0; - const total_ms = @divTrunc(total_ns, std.time.ns_per_ms); - try job_span.pushAttribute(try Attribute.initInt(self.allocator, "job.total_duration_ms", @as(i64, @intCast(total_ms)))); + fn recordStepJobParked(self: *RequestRecord, event: telemetry.StepJobParkedEvent) !void { + // Look up JobState and append ParkEpisode + if (self.job_states.getPtr(event.job_ctx)) |job_state| { + var episode = try ParkEpisode.init( + self.allocator, + event.cause, + event.token, + @as(i64, @intCast(event.timestamp_ms)), + ); + episode.concurrency_limit_current = event.concurrency_limit_current; + episode.concurrency_limit_max = event.concurrency_limit_max; + errdefer episode.deinit(); - _ = self.step_job_spans.remove(event.job_ctx); + try job_state.park_episodes.append(self.allocator, episode); + } + + // Record event on root request span + var request_event = try self.buildStepJobStageEvent( + "zerver.step_job_parked", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "parked", + null, + null, + ); + errdefer request_event.deinit(); + try request_event.addAttribute(try Attribute.initString(self.allocator, "job.park_cause", event.cause)); + if (event.token) |tok| { + try request_event.addAttribute(try Attribute.initInt(self.allocator, "job.park_token", @as(i64, @intCast(tok)))); + } + if (event.concurrency_limit_current) |curr| { + try request_event.addAttribute(try Attribute.initInt(self.allocator, "job.concurrency_current", @as(i64, @intCast(curr)))); + } + if (event.concurrency_limit_max) |max| { + try request_event.addAttribute(try Attribute.initInt(self.allocator, "job.concurrency_max", @as(i64, @intCast(max)))); + } + try self.pushEvent(request_event); } - fn recordStepWait(self: *RequestRecord, event: telemetry.StepWaitEvent) !void { - var req_event = try RequestEvent.init(self.allocator, "zerver.step_wait", event.timestamp_ms * std.time.ns_per_ms); - var committed = false; - defer if (!committed) req_event.deinit(); - try req_event.addAttribute(try Attribute.initString(self.allocator, "job.type", "step")); - try req_event.addAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(event.need_sequence)))); - try req_event.addAttribute(try Attribute.initString(self.allocator, "job.stage", "waiting")); - try self.pushEvent(req_event); - committed = true; + fn recordStepJobResumed(self: *RequestRecord, event: telemetry.StepJobResumedEvent) !void { + // Look up JobState and update last ParkEpisode resume_ts + if (self.job_states.getPtr(event.job_ctx)) |job_state| { + // Find the last park episode (most recent one without resume_ts) + var i: usize = job_state.park_episodes.items.len; + while (i > 0) { + i -= 1; + if (job_state.park_episodes.items[i].resume_ts == null) { + job_state.park_episodes.items[i].resume_ts = @as(i64, @intCast(event.timestamp_ms)); + break; + } + } + } + + // Record event on root request span + var request_event = try self.buildStepJobStageEvent( + "zerver.step_job_resumed", + event.timestamp_ms, + event.need_sequence, + event.job_ctx, + event.queue, + "resumed", + null, + null, + ); + errdefer request_event.deinit(); + try self.pushEvent(request_event); } fn removeActiveStep(self: *RequestRecord, span: *ChildSpan) void { @@ -841,33 +1322,9 @@ const RequestRecord = struct { } } - var job_iter = self.job_spans.iterator(); - while (job_iter.next()) |entry| { - const span = entry.value_ptr.*; - if (span.end_time_unix_ns <= span.start_time_unix_ns) { - span.end_time_unix_ns = end_time_unix_ns; - } - if (span.status != .@"error") { - span.setStatus(.@"error", "job span incomplete") catch {}; - } - } - - var step_job_iter = self.step_job_spans.iterator(); - while (step_job_iter.next()) |entry| { - const span = entry.value_ptr.*; - if (span.end_time_unix_ns <= span.start_time_unix_ns) { - span.end_time_unix_ns = end_time_unix_ns; - } - if (span.status != .@"error") { - span.setStatus(.@"error", "step job span incomplete") catch {}; - } - } - self.step_stack.clearRetainingCapacity(); self.step_spans.clearRetainingCapacity(); self.effect_spans.clearRetainingCapacity(); - self.job_spans.clearRetainingCapacity(); - self.step_job_spans.clearRetainingCapacity(); } fn applyRequestEnd(self: *RequestRecord, event: telemetry.RequestEndEvent) !void { @@ -878,13 +1335,13 @@ const RequestRecord = struct { if (event.response_content_type.len != 0) { self.allocator.free(self.response_content_type); self.response_content_type = try self.allocator.dupe(u8, event.response_content_type); - try self.pushAttribute(try Attribute.initString(self.allocator, "http.response_content_type", event.response_content_type)); + try self.pushAttribute(try Attribute.initString(self.allocator, "http.response.header.content-type", event.response_content_type)); } self.response_body_bytes = event.response_body_bytes; self.response_streaming = event.response_streaming; - try self.pushAttribute(try Attribute.initInt(self.allocator, "http.response_content_length", @as(i64, @intCast(event.response_body_bytes)))); + try self.pushAttribute(try Attribute.initInt(self.allocator, "http.response.body.size", @as(i64, @intCast(event.response_body_bytes)))); try self.pushAttribute(try Attribute.initBool(self.allocator, "zerver.response_streaming", event.response_streaming)); - try self.pushAttribute(try Attribute.initInt(self.allocator, "http.status_code", @as(i64, @intCast(event.status_code)))); + try self.pushAttribute(try Attribute.initInt(self.allocator, "http.response.status_code", @as(i64, @intCast(event.status_code)))); try self.pushAttribute(try Attribute.initString(self.allocator, "zerver.outcome", event.outcome)); if (event.error_ctx) |ctx| { @@ -912,103 +1369,6 @@ const RequestRecord = struct { self.status_message = try self.allocator.dupe(u8, message); } - fn ensureJobSpan( - self: *RequestRecord, - effect_parent: ?*ChildSpan, - need_sequence: usize, - effect_sequence: usize, - queue: []const u8, - timestamp_ms: u64, - ) !*ChildSpan { - if (self.job_spans.get(effect_sequence)) |span| { - const timestamp_ns = timestamp_ms * std.time.ns_per_ms; - if (timestamp_ns < span.start_time_unix_ns) { - span.start_time_unix_ns = timestamp_ns; - } - return span; - } - - const parent_id: [8]u8 = if (effect_parent) |parent| - parent.span_id - else - self.span_id; - const start_ns = timestamp_ms * std.time.ns_per_ms; - - var span = try ChildSpan.create(self.allocator, "effect_job", .internal, parent_id, start_ns); - errdefer { - span.deinit(); - self.allocator.destroy(span); - } - - try span.pushAttribute(try Attribute.initString(self.allocator, "job.type", "effect")); - try span.pushAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); - try span.pushAttribute(try Attribute.initInt(self.allocator, "job.effect_sequence", @as(i64, @intCast(effect_sequence)))); - try span.pushAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); - - self.child_spans.append(self.allocator, span) catch |err| { - span.deinit(); - self.allocator.destroy(span); - return err; - }; - - self.job_spans.put(effect_sequence, span) catch |err| { - _ = self.child_spans.pop(); - span.deinit(); - self.allocator.destroy(span); - return err; - }; - - return span; - } - - fn ensureStepJobSpan( - self: *RequestRecord, - need_sequence: usize, - job_ctx: usize, - queue: []const u8, - timestamp_ms: u64, - ) !*ChildSpan { - if (self.step_job_spans.get(job_ctx)) |span| { - const timestamp_ns = timestamp_ms * std.time.ns_per_ms; - if (timestamp_ns < span.start_time_unix_ns) { - span.start_time_unix_ns = timestamp_ns; - } - return span; - } - - const parent_id: [8]u8 = if (self.step_stack.items.len != 0) - self.step_stack.items[self.step_stack.items.len - 1].span_id - else - self.span_id; - const start_ns = timestamp_ms * std.time.ns_per_ms; - - var span = try ChildSpan.create(self.allocator, "step_job", .internal, parent_id, start_ns); - errdefer { - span.deinit(); - self.allocator.destroy(span); - } - - try span.pushAttribute(try Attribute.initString(self.allocator, "job.type", "step")); - try span.pushAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); - try span.pushAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(job_ctx)))); - try span.pushAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); - - self.child_spans.append(self.allocator, span) catch |err| { - span.deinit(); - self.allocator.destroy(span); - return err; - }; - - self.step_job_spans.put(job_ctx, span) catch |err| { - _ = self.child_spans.pop(); - span.deinit(); - self.allocator.destroy(span); - return err; - }; - - return span; - } - fn buildJobStageEvent( self: *RequestRecord, name: []const u8, @@ -1022,8 +1382,8 @@ const RequestRecord = struct { var event = try RequestEvent.init(self.allocator, name, timestamp_ms * std.time.ns_per_ms); errdefer event.deinit(); try event.addAttribute(try Attribute.initString(self.allocator, "job.type", "effect")); - try event.addAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); - try event.addAttribute(try Attribute.initInt(self.allocator, "job.effect_sequence", @as(i64, @intCast(effect_sequence)))); + try event.addAttribute(try Attribute.initInt(self.allocator, "effect.sequence", @as(i64, @intCast(need_sequence)))); + try event.addAttribute(try Attribute.initInt(self.allocator, "job.effect.sequence", @as(i64, @intCast(effect_sequence)))); try event.addAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); try event.addAttribute(try Attribute.initString(self.allocator, "job.stage", stage)); if (success) |value| { @@ -1046,8 +1406,8 @@ const RequestRecord = struct { var event = try RequestEvent.init(self.allocator, name, timestamp_ms * std.time.ns_per_ms); errdefer event.deinit(); try event.addAttribute(try Attribute.initString(self.allocator, "job.type", "step")); - try event.addAttribute(try Attribute.initInt(self.allocator, "job.need_sequence", @as(i64, @intCast(need_sequence)))); - try event.addAttribute(try Attribute.initInt(self.allocator, "job.ctx", @as(i64, @intCast(job_ctx)))); + try event.addAttribute(try Attribute.initInt(self.allocator, "need.sequence", @as(i64, @intCast(need_sequence)))); + try event.addAttribute(try Attribute.initInt(self.allocator, "job.step.ctx", @as(i64, @intCast(job_ctx)))); try event.addAttribute(try Attribute.initString(self.allocator, "job.queue", queue)); try event.addAttribute(try Attribute.initString(self.allocator, "job.stage", stage)); if (worker_index) |worker| { @@ -1278,6 +1638,36 @@ pub const OtelExporter = struct { try record.recordStepWait(payload); } }, + .effect_job_taken => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordEffectJobTaken(payload); + } + }, + .effect_job_parked => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordEffectJobParked(payload); + } + }, + .effect_job_resumed => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordEffectJobResumed(payload); + } + }, + .step_job_taken => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordStepJobTaken(payload); + } + }, + .step_job_parked => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordStepJobParked(payload); + } + }, + .step_job_resumed => |payload| { + if (self.requests.get(payload.request_id)) |record| { + try record.recordStepJobResumed(payload); + } + }, } if (to_export) |record| { diff --git a/src/zerver/observability/otel_config.zig b/src/zerver/observability/otel_config.zig new file mode 100644 index 0000000..6767ba0 --- /dev/null +++ b/src/zerver/observability/otel_config.zig @@ -0,0 +1,85 @@ +const std = @import("std"); + +/// Configuration for OpenTelemetry behavior, primarily controlling span promotion thresholds. +pub const OtelConfig = struct { + /// Minimum queue wait time (ms) before promoting to a dedicated span. + promote_queue_ms: u32, + + /// Minimum park duration (ms) before promoting to a dedicated span. + promote_park_ms: u32, + + /// Force all job spans to be created (debug mode). + debug_jobs: bool, + + /// Name of the effects queue. + queue_name_effects: []const u8, + + /// Name of the continuations queue. + queue_name_cont: []const u8, + + /// Whether to export job queue depth metrics. + export_job_depth: bool, + + /// Initialize config from environment variables with defaults. + pub fn init(allocator: std.mem.Allocator) OtelConfig { + return .{ + .promote_queue_ms = parseEnvU32("ZER_VER_PROMOTE_QUEUE_MS", 5), + .promote_park_ms = parseEnvU32("ZER_VER_PROMOTE_PARK_MS", 5), + .debug_jobs = parseEnvBool("ZER_VER_DEBUG_JOBS", false), + .queue_name_effects = parseEnvString(allocator, "ZER_VER_QUEUE_NAME_EFFECTS", "effects"), + .queue_name_cont = parseEnvString(allocator, "ZER_VER_QUEUE_NAME_CONT", "continuations"), + .export_job_depth = parseEnvBool("ZER_VER_EXPORT_JOB_DEPTH", false), + }; + } + + /// Clean up allocated strings if any. + pub fn deinit(self: *OtelConfig, allocator: std.mem.Allocator) void { + // Only free if the string was allocated (not the default literal) + const default_effects = "effects"; + const default_cont = "continuations"; + if (self.queue_name_effects.ptr != default_effects.ptr) { + allocator.free(self.queue_name_effects); + } + if (self.queue_name_cont.ptr != default_cont.ptr) { + allocator.free(self.queue_name_cont); + } + } +}; + +/// Parse environment variable as u32, return default if not found or invalid. +fn parseEnvU32(key: []const u8, default: u32) u32 { + const value = std.posix.getenv(key) orelse return default; + return std.fmt.parseInt(u32, value, 10) catch default; +} + +/// Parse environment variable as bool (1=true, 0=false), return default if not found. +fn parseEnvBool(key: []const u8, default: bool) bool { + const value = std.posix.getenv(key) orelse return default; + if (std.mem.eql(u8, value, "1") or std.mem.eql(u8, value, "true")) { + return true; + } + if (std.mem.eql(u8, value, "0") or std.mem.eql(u8, value, "false")) { + return false; + } + return default; +} + +/// Parse environment variable as string, return default if not found. +/// Caller owns the returned memory if it's not the default. +fn parseEnvString(allocator: std.mem.Allocator, key: []const u8, default: []const u8) []const u8 { + const value = std.posix.getenv(key) orelse return default; + // Allocate and return copy to ensure consistent ownership + return allocator.dupe(u8, value) catch default; +} + +test "OtelConfig defaults" { + var config = OtelConfig.init(std.testing.allocator); + defer config.deinit(std.testing.allocator); + + try std.testing.expectEqual(@as(u32, 5), config.promote_queue_ms); + try std.testing.expectEqual(@as(u32, 5), config.promote_park_ms); + try std.testing.expectEqual(false, config.debug_jobs); + try std.testing.expectEqualStrings("effects", config.queue_name_effects); + try std.testing.expectEqualStrings("continuations", config.queue_name_cont); + try std.testing.expectEqual(false, config.export_job_depth); +} diff --git a/src/zerver/observability/telemetry.zig b/src/zerver/observability/telemetry.zig index d359a1b..715f539 100644 --- a/src/zerver/observability/telemetry.zig +++ b/src/zerver/observability/telemetry.zig @@ -200,6 +200,70 @@ pub const StepWaitEvent = struct { timestamp_ms: u64, }; +/// Fired when an effect job is dequeued from the job system (taken by a worker). +pub const EffectJobTakenEvent = struct { + request_id: []const u8, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + worker_index: usize, + timestamp_ms: u64, +}; + +/// Fired when an effect job is parked (waiting on I/O, rate limit, etc.). +pub const EffectJobParkedEvent = struct { + request_id: []const u8, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + cause: []const u8, // io_wait|rate_limit|backpressure|lock|timer|other + token: ?u32, + concurrency_limit_current: ?usize, + concurrency_limit_max: ?usize, + timestamp_ms: u64, +}; + +/// Fired when an effect job is resumed after parking. +pub const EffectJobResumedEvent = struct { + request_id: []const u8, + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + timestamp_ms: u64, +}; + +/// Fired when a step job is dequeued from the job system (taken by a worker). +pub const StepJobTakenEvent = struct { + request_id: []const u8, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: usize, + timestamp_ms: u64, +}; + +/// Fired when a step job is parked (waiting on continuation). +pub const StepJobParkedEvent = struct { + request_id: []const u8, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + cause: []const u8, + token: ?u32, + concurrency_limit_current: ?usize, + concurrency_limit_max: ?usize, + timestamp_ms: u64, +}; + +/// Fired when a step job is resumed after parking. +pub const StepJobResumedEvent = struct { + request_id: []const u8, + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + timestamp_ms: u64, +}; + /// Union of all telemetry signals publishable to subscribers. pub const Event = union(enum) { request_start: RequestStartEvent, @@ -218,6 +282,12 @@ pub const Event = union(enum) { step_job_started: StepJobStartedEvent, step_job_completed: StepJobCompletedEvent, step_wait: StepWaitEvent, + effect_job_taken: EffectJobTakenEvent, + effect_job_parked: EffectJobParkedEvent, + effect_job_resumed: EffectJobResumedEvent, + step_job_taken: StepJobTakenEvent, + step_job_parked: StepJobParkedEvent, + step_job_resumed: StepJobResumedEvent, }; /// Options supplied when building per-request telemetry. @@ -716,6 +786,90 @@ pub const Telemetry = struct { ); } + pub const EffectJobTakenDetails = struct { + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + worker_index: usize, + }; + + pub fn effectJobTaken(self: *Telemetry, details: EffectJobTakenDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("effect_job_taken", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("effect_sequence", details.effect_sequence), + slog.Attr.string("queue", details.queue), + slog.Attr.uint("worker_index", details.worker_index), + }); + + self.emit(.{ .effect_job_taken = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .effect_sequence = details.effect_sequence, + .queue = details.queue, + .worker_index = details.worker_index, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + } + + pub const EffectJobParkedDetails = struct { + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + cause: []const u8, + token: ?u32 = null, + concurrency_limit_current: ?usize = null, + concurrency_limit_max: ?usize = null, + }; + + pub fn effectJobParked(self: *Telemetry, details: EffectJobParkedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("effect_job_parked", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("effect_sequence", details.effect_sequence), + slog.Attr.string("queue", details.queue), + slog.Attr.string("cause", details.cause), + }); + + self.emit(.{ .effect_job_parked = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .effect_sequence = details.effect_sequence, + .queue = details.queue, + .cause = details.cause, + .token = details.token, + .concurrency_limit_current = details.concurrency_limit_current, + .concurrency_limit_max = details.concurrency_limit_max, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + } + + pub const EffectJobResumedDetails = struct { + need_sequence: usize, + effect_sequence: usize, + queue: []const u8, + }; + + pub fn effectJobResumed(self: *Telemetry, details: EffectJobResumedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("effect_job_resumed", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("effect_sequence", details.effect_sequence), + slog.Attr.string("queue", details.queue), + }); + + self.emit(.{ .effect_job_resumed = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .effect_sequence = details.effect_sequence, + .queue = details.queue, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + } + pub const StepJobEnqueuedDetails = struct { need_sequence: usize, job_ctx: usize, @@ -801,6 +955,90 @@ pub const Telemetry = struct { self.tracer.recordStepJobCompleted(details.need_sequence, details.job_ctx, details.queue, details.worker_index, details.decision); } + pub const StepJobTakenDetails = struct { + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + worker_index: usize, + }; + + pub fn stepJobTaken(self: *Telemetry, details: StepJobTakenDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("step_job_taken", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("job_ctx", @as(u64, @intCast(details.job_ctx))), + slog.Attr.string("queue", details.queue), + slog.Attr.uint("worker_index", details.worker_index), + }); + + self.emit(.{ .step_job_taken = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .job_ctx = details.job_ctx, + .queue = details.queue, + .worker_index = details.worker_index, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + } + + pub const StepJobParkedDetails = struct { + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + cause: []const u8, + token: ?u32 = null, + concurrency_limit_current: ?usize = null, + concurrency_limit_max: ?usize = null, + }; + + pub fn stepJobParked(self: *Telemetry, details: StepJobParkedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("step_job_parked", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("job_ctx", @as(u64, @intCast(details.job_ctx))), + slog.Attr.string("queue", details.queue), + slog.Attr.string("cause", details.cause), + }); + + self.emit(.{ .step_job_parked = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .job_ctx = details.job_ctx, + .queue = details.queue, + .cause = details.cause, + .token = details.token, + .concurrency_limit_current = details.concurrency_limit_current, + .concurrency_limit_max = details.concurrency_limit_max, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + } + + pub const StepJobResumedDetails = struct { + need_sequence: usize, + job_ctx: usize, + queue: []const u8, + }; + + pub fn stepJobResumed(self: *Telemetry, details: StepJobResumedDetails) void { + const timestamp_ms = std.time.milliTimestamp(); + self.logDebug("step_job_resumed", &.{ + slog.Attr.string("request_id", self.request_id), + slog.Attr.uint("need_sequence", details.need_sequence), + slog.Attr.uint("job_ctx", @as(u64, @intCast(details.job_ctx))), + slog.Attr.string("queue", details.queue), + }); + + self.emit(.{ .step_job_resumed = .{ + .request_id = self.request_id, + .need_sequence = details.need_sequence, + .job_ctx = details.job_ctx, + .queue = details.queue, + .timestamp_ms = @as(u64, @intCast(timestamp_ms)), + } }); + } + pub fn stepWait(self: *Telemetry, need_sequence: usize) void { const timestamp_ms = std.time.milliTimestamp(); self.logDebug("step_wait", &.{ diff --git a/src/zerver/runtime/global.zig b/src/zerver/runtime/global.zig index c8d374e..f7d080b 100644 --- a/src/zerver/runtime/global.zig +++ b/src/zerver/runtime/global.zig @@ -1,6 +1,7 @@ const resources_mod = @import("resources.zig"); var global_resources: ?*resources_mod.RuntimeResources = null; +// TODO: Concurrency - global_resources has no synchronization, so concurrent set/clear/get can race; guard with atomics or a mutex. pub fn set(resources: *resources_mod.RuntimeResources) void { global_resources = resources; diff --git a/src/zerver/runtime/handler.zig b/src/zerver/runtime/handler.zig index 540f96e..baeeec1 100644 --- a/src/zerver/runtime/handler.zig +++ b/src/zerver/runtime/handler.zig @@ -28,13 +28,17 @@ pub fn readRequestWithTimeout( timeout_ms: u32, ) ![]u8 { var req_buf = std.ArrayList(u8).initCapacity(allocator, 4096) catch unreachable; + // TODO: Leak - add errdefer req_buf.deinit(allocator) so early error paths don't orphan the buffer allocations. // TODO: Safety - Replace 'catch unreachable' with proper error propagation or handling for allocation failures in readRequestWithTimeout to prevent crashes. + // TODO: Safety - Replace 'catch unreachable' with proper error propagation or handling for allocation failures in readRequestWithTimeout to prevent crashes. + // TODO: RFC 9110 Section 5.4 - The 'max_size' (4096 bytes) in readRequestWithTimeout is an arbitrary limit for headers. If headers exceed this, it should result in a 413 "Content Too Large" or 431 "Request Header Fields Too Large" error. // TODO: Logical Error - The 'max_size' (4096 bytes) in readRequestWithTimeout is an arbitrary limit for headers. If headers exceed this, it results in 'error.InvalidRequest'. Consider handling this as a '413 Payload Too Large' or a more specific error, and ensure this limit is configurable or documented. var read_buf: [256]u8 = undefined; const max_size = 4096; const start_time = std.time.milliTimestamp(); + // TODO: RFC 9110 Section 5.5 - The parser should reject messages containing CTL characters like CR, LF, or NUL within field values to prevent request smuggling attacks. // Phase 1: Read headers until \r\n\r\n var headers_complete = false; while (req_buf.items.len < max_size and !headers_complete) { @@ -218,6 +222,7 @@ fn readChunkedBody( if (chunk_size == 0) { // Last chunk - read until we have the final \r\n\r\n (trailers + final CRLF) + // TODO: RFC 9112 Section 7.1.2 - Implement parsing of trailer fields. The current implementation reads until the final CRLF but does not process the trailers. while (!std.mem.endsWith(u8, req_buf.items, "\r\n\r\n")) { const bytes_read = try readWithTimeout(connection, &read_buf, timeout_ms, start_time); if (bytes_read == 0) return error.ConnectionClosed; @@ -246,6 +251,7 @@ fn readChunkedBody( // TODO: Bug - After consuming this chunk we never advance body_start/req_buf to the next chunk, // so multi-chunk payloads keep reprocessing the same data and will spin or time out. + // TODO: RFC 9112 Section 7.1 - A malformed chunk size line (e.g., empty) should be treated as an error rather than causing a potential infinite loop. } } @@ -324,6 +330,7 @@ pub fn sendResponse( response: []const u8, ) !void { // TODO: RFC 9110/9112 - Ensure proper HTTP/1.1 message framing for responses, including support for Transfer-Encoding (e.g., chunked encoding) if applicable (RFC 9112 Section 6). + // TODO: RFC 9112 Section 6 - This function should automatically handle response framing by adding Content-Length or Transfer-Encoding: chunked headers based on the response body. // TODO: SSE - Implement a mechanism for streaming responses, allowing incremental writing of data for Server-Sent Events (HTML Living Standard). const preview_len = @min(response.len, 120); slog.debug("Sending HTTP response", &.{ @@ -336,6 +343,7 @@ pub fn sendResponse( slog.Attr.string("error", @errorName(err)), }); // TODO: Bug - We swallow the write failure and still report success to callers, leaving them unaware that the response never went out. + // TODO: Bug - We swallow the write failure and still report success to callers, leaving them unaware that the response never went out. }; } @@ -365,6 +373,8 @@ pub fn sendErrorResponse( status: []const u8, message: []const u8, ) !void { + // TODO: Safety/Memory - The fixed-size buffer in sendErrorResponse might lead to truncation or errors for long status/message strings. Consider using an allocator for dynamic sizing. + // TODO: Safety/Memory - The fixed-size buffer in sendErrorResponse might lead to truncation or errors for long status/message strings. Consider using an allocator for dynamic sizing. // TODO: Safety/Memory - The fixed-size buffer in sendErrorResponse might lead to truncation or errors for long status/message strings. Consider using an allocator for dynamic sizing. var buf: [512]u8 = undefined; const response = try std.fmt.bufPrint(&buf, "HTTP/1.1 {s}\r\nContent-Type: text/plain\r\nContent-Length: {d}\r\n\r\n{s}", .{ diff --git a/src/zerver/runtime/listener.zig b/src/zerver/runtime/listener.zig index 9eaa561..614c123 100644 --- a/src/zerver/runtime/listener.zig +++ b/src/zerver/runtime/listener.zig @@ -13,6 +13,7 @@ pub fn listenAndServe( allocator: std.mem.Allocator, ) !void { const server_addr = try std.net.Address.parseIp("127.0.0.1", 8080); + // TODO: Bug - Hard-coding 127.0.0.1:8080 ignores runtime configuration; use server Config listen address instead. var listener = try server_addr.listen(.{ .reuse_address = true, }); @@ -34,6 +35,7 @@ pub fn listenAndServe( // Handle persistent connection - RFC 9112 Section 9 // TODO: Bug - Propagating a single connection error through this `try` will tear down the entire listener loop instead of just dropping the bad client; swallow and continue instead. + // TODO: Bug - Propagating a single connection error through this `try` will tear down the entire listener loop instead of just dropping the bad client; swallow and continue instead. try handleConnection(srv, allocator, connection); } } @@ -121,6 +123,7 @@ fn handleConnection( try handler.sendStreamingResponse(connection, streaming_resp.headers, streaming_resp.writer, streaming_resp.context); // For streaming responses, we typically don't keep the connection alive in the same way // as the stream may run indefinitely + // TODO: RFC 9112 Section 8.1 - Pipelining is not supported. The server should read and respond to requests sequentially. // TODO: Logical Error - For streaming responses (e.g., SSE), the 'shouldKeepConnectionAlive' check is currently skipped. Ensure streaming connections are properly managed for persistence, as they are typically long-lived. return; }, diff --git a/tests/integration/rfc9112_message_format_test.zig b/tests/integration/rfc9112_message_format_test.zig new file mode 100644 index 0000000..af64845 --- /dev/null +++ b/tests/integration/rfc9112_message_format_test.zig @@ -0,0 +1,130 @@ +const std = @import("std"); +const zerver = @import("../../src/zerver/root.zig"); +const test_harness = @import("test_harness.zig"); + +test "Request Line - Valid" { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + defer _ = gpa.deinit(); + + var server = try test_harness.createTestServer(allocator); + defer server.deinit(); + + try server.addRoute(.GET, "/test", .{ + .steps = &.{ + zerver.step("test", struct { + fn handler(ctx: *zerver.Ctx) !zerver.Decision { + _ = ctx; + return zerver.done(.{ .body = .{ .complete = "ok" } }); + } + }.handler), + }, + }); + + const request_text = + \GET /test HTTP/1.1 + + \Host: localhost + + \ + + ; + + const response_text = try server.handleRequest(request_text, allocator); + try std.testing.expect(std.mem.startsWith(u8, response_text, "HTTP/1.1 200 OK")); +} + +test "Request Line - Invalid Method" { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + defer _ = gpa.deinit(); + + var server = try test_harness.createTestServer(allocator); + defer server.deinit(); + + const request_text = + \INVALID /test HTTP/1.1 + + \Host: localhost + + \ + + ; + + const response_text = try server.handleRequest(request_text, allocator); + try std.testing.expect(std.mem.startsWith(u8, response_text, "HTTP/1.1 400 Bad Request")); +} + +test "Request Line - Missing Path" { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + defer _ = gpa.deinit(); + + var server = try test_harness.createTestServer(allocator); + defer server.deinit(); + + const request_text = + \GET HTTP/1.1 + + \Host: localhost + + \ + + ; + + const response_text = try server.handleRequest(request_text, allocator); + try std.testing.expect(std.mem.startsWith(u8, response_text, "HTTP/1.1 400 Bad Request")); +} + +test "Request Line - Missing Version" { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + defer _ = gpa.deinit(); + + var server = try test_harness.createTestServer(allocator); + defer server.deinit(); + + const request_text = + \GET /test + + \Host: localhost + + \ + + ; + + const response_text = try server.handleRequest(request_text, allocator); + try std.testing.expect(std.mem.startsWith(u8, response_text, "HTTP/1.1 400 Bad Request")); +} + +test "Request Line - Extra Whitespace" { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + const allocator = gpa.allocator(); + defer _ = gpa.deinit(); + + var server = try test_harness.createTestServer(allocator); + defer server.deinit(); + + try server.addRoute(.GET, "/test", .{ + .steps = &.{ + zerver.step("test", struct { + fn handler(ctx: *zerver.Ctx) !zerver.Decision { + _ = ctx; + return zerver.done(.{ .body = .{ .complete = "ok" } }); + } + }.handler), + }, + }); + + const request_text = + \GET /test HTTP/1.1 + + \Host: localhost + + \ + + ; + + const response_text = try server.handleRequest(request_text, allocator); + try std.testing.expect(std.mem.startsWith(u8, response_text, "HTTP/1.1 200 OK")); +}