diff --git a/docs/design/architecture.rst b/docs/design/architecture.rst index 9b6cc46f..49321291 100644 --- a/docs/design/architecture.rst +++ b/docs/design/architecture.rst @@ -33,7 +33,7 @@ Veeksha is composed of several key components that work together: │ ▼ │ │ ┌──────────────────────────────────────────┐ │ │ │ Client Runners │ │ - │ │ - Async HTTP clients (httpx) │ │ + │ │ - Async HTTP clients (aiohttp) │ │ │ │ - Streaming response handling │ │ │ └──────────────────────────────────────────┘ │ └─────────────────────────────────────────────────────────────────────────┘ @@ -140,8 +140,8 @@ Veeksha uses a multi-threaded architecture with configurable worker counts: and feed results to the evaluator. More threads help with high throughput. **Client Threads** - Each runs an async event loop with an ``httpx.AsyncClient`` for making - concurrent HTTP requests. More threads increase I/O parallelism. + Each runs an async event loop with its own ``aiohttp.ClientSession`` for + making concurrent HTTP requests. More threads increase I/O parallelism. .. note:: diff --git a/pyproject.toml b/pyproject.toml index 07e015d9..7cce90a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "textual-plotext", "requests", "uvloop", - "httpx", + "aiohttp", "nvidia-ml-py", "rich", "vidhi", diff --git a/tests/unit/client/test_vajra_tts_stream_client.py b/tests/unit/client/test_vajra_tts_stream_client.py index dae03874..f08cd175 100644 --- a/tests/unit/client/test_vajra_tts_stream_client.py +++ b/tests/unit/client/test_vajra_tts_stream_client.py @@ -210,7 +210,7 @@ def _run_request( monkeypatch: pytest.MonkeyPatch, events: list[str], ): - monkeypatch.setattr(client, "_connect", lambda: _FakeConnection(websocket)) + monkeypatch.setattr(client, "_connect", lambda *a, **k: _FakeConnection(websocket)) return asyncio.run( client.send_request( _request(), diff --git a/tests/unit/evaluator/performance/test_lifecycle_drift.py b/tests/unit/evaluator/performance/test_lifecycle_drift.py new file mode 100644 index 00000000..f7a53ebf --- /dev/null +++ b/tests/unit/evaluator/performance/test_lifecycle_drift.py @@ -0,0 +1,82 @@ +"""Unit tests for harness request-lifecycle drift metrics + soft WARN.""" + +import logging +from types import SimpleNamespace + +import pytest + +from veeksha.config.evaluator import PerformanceEvaluatorConfig +from veeksha.evaluator.performance.base import PerformanceEvaluator + + +def _response(ready, dispatched, pickup, sent): + return SimpleNamespace( + scheduler_ready_at=ready, + scheduler_dispatched_at=dispatched, + client_picked_up_at=pickup, + client_sent_at=sent, + ) + + +def _evaluator(threshold_ms=5.0): + return PerformanceEvaluator( + PerformanceEvaluatorConfig(lifecycle_drift_warn_threshold_ms=threshold_ms) + ) + + +@pytest.mark.unit +def test_lifecycle_metrics_in_summary(): + ev = _evaluator() + for _ in range(20): + # ready->dispatch 1ms, dispatch->pickup 0.5ms, pickup->send 2ms + ev._accumulate_lifecycle_drift(_response(100.0, 100.001, 100.0015, 100.0035)) + + summary = ev.get_aggregated_summary() + assert "Harness Ready-to-Dispatch (ms) (P99)" in summary + assert summary["Harness Ready-to-Dispatch (ms) (P99)"] == pytest.approx( + 1.0, rel=0.05 + ) + assert summary["Harness Pickup-to-Send (ms) (P99)"] == pytest.approx(2.0, rel=0.05) + assert summary["Harness Ready-to-Send (ms) (P99)"] == pytest.approx(3.5, rel=0.05) + + +@pytest.mark.unit +def test_missing_timestamps_are_skipped(): + ev = _evaluator() + # client_sent_at missing -> pickup->send and ready->send have no samples + ev._accumulate_lifecycle_drift(_response(100.0, 100.001, 100.0015, None)) + assert ev._lifecycle_sketches["Harness Ready-to-Dispatch (ms)"].sketch.count == 1 + assert ev._lifecycle_sketches["Harness Pickup-to-Send (ms)"].sketch.count == 0 + assert ev._lifecycle_sketches["Harness Ready-to-Send (ms)"].sketch.count == 0 + + +@pytest.mark.unit +def test_warn_fires_above_threshold(caplog): + ev = _evaluator(threshold_ms=5.0) + for _ in range(30): + # pickup->send = 20ms, well above the 5ms threshold + ev._accumulate_lifecycle_drift(_response(100.0, 100.001, 100.0015, 100.0215)) + with caplog.at_level(logging.WARNING): + ev._warn_on_lifecycle_drift() + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("Pickup-to-Send" in w and "lifecycle drift high" in w for w in warnings) + + +@pytest.mark.unit +def test_no_warn_within_threshold(caplog): + ev = _evaluator(threshold_ms=50.0) + for _ in range(30): + ev._accumulate_lifecycle_drift(_response(100.0, 100.001, 100.0015, 100.0035)) + with caplog.at_level(logging.WARNING): + ev._warn_on_lifecycle_drift() + assert not [r for r in caplog.records if "lifecycle drift high" in r.getMessage()] + + +@pytest.mark.unit +def test_warn_disabled_when_threshold_nonpositive(caplog): + ev = _evaluator(threshold_ms=0.0) + for _ in range(30): + ev._accumulate_lifecycle_drift(_response(100.0, 100.001, 100.0015, 100.5)) + with caplog.at_level(logging.WARNING): + ev._warn_on_lifecycle_drift() + assert not [r for r in caplog.records if "lifecycle drift high" in r.getMessage()] diff --git a/tests/unit/preflight/__init__.py b/tests/unit/preflight/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/preflight/test_chat_mock_integration.py b/tests/unit/preflight/test_chat_mock_integration.py new file mode 100644 index 00000000..49f7cd27 --- /dev/null +++ b/tests/unit/preflight/test_chat_mock_integration.py @@ -0,0 +1,104 @@ +"""End-to-end: real chat client -> spawned mock server -> scorer. + +Exercises the actual OpenAIChatCompletionsClient streaming path against a +deterministic mock server running in a separate process, then scores the +paired client/server timestamps. This is the vertical slice's proof that +request/response delivery lag and server pacing fidelity are measured correctly. +""" + +import asyncio + +import pytest + +from veeksha.client import ClientRegistry +from veeksha.config.client import OpenAIChatCompletionsClientConfig +from veeksha.core.request import Request +from veeksha.core.request_content import TextChannelRequestContent +from veeksha.core.tokenizer import TokenizerHandle, TokenizerProvider +from veeksha.preflight import scorer +from veeksha.preflight.spawn import spawn_mock_chat_server +from veeksha.types import ChannelModality + +TTFC_MS = 80.0 +TPOC_MS = 10.0 +NUM_CHUNKS = 24 + + +def _word_tokenizer_provider() -> TokenizerProvider: + handle = TokenizerHandle( + count_tokens=lambda text: len(str(text).split()), + decode=lambda token_ids: "", + encode=lambda text: [0] * len(str(text).split()), + ) + return TokenizerProvider({ChannelModality.TEXT: handle}) + + +def _make_request(rid: int) -> Request: + return Request( + id=rid, + channels={ + ChannelModality.TEXT: TextChannelRequestContent(input_text=f"hello {rid}") + }, + ) + + +@pytest.mark.unit +def test_chat_client_against_mock_is_measured(): + with spawn_mock_chat_server( + ttfc_ms=TTFC_MS, tpoc_ms=TPOC_MS, num_chunks=NUM_CHUNKS + ) as server: + config = OpenAIChatCompletionsClientConfig( + api_base=server.api_base, + api_key="preflight", + model="dummy", + record_preflight_timing=True, + ) + client = ClientRegistry.get( + config.get_type(), + config=config, + tokenizer_provider=_word_tokenizer_provider(), + ) + + async def _run(): + results = [] + for rid in range(6): + results.append( + await client.send_request(request=_make_request(rid), session_id=0) + ) + await client.aclose() + return results + + results = asyncio.run(_run()) + server_records = server.fetch_records() + + # every request succeeded and streamed all chunks + assert all(r.success for r in results), [r.error_msg for r in results] + assert all(len(r.chunk_recv_times) == NUM_CHUNKS for r in results) + assert len(server_records) == len(results) + + report = scorer.score(results, server_records, ttfc_ms=TTFC_MS, tpoc_ms=TPOC_MS) + + # all requests paired with a server record + assert report.n_paired_requests == len(results) + assert report.unpaired_fraction == 0.0 + + m = report.metrics + # delivery lags exist, are non-negative, and small on loopback + rd = m[scorer.M_REQUEST_DELIVERY] + resp = m[scorer.M_RESPONSE_DELIVERY] + assert rd.count == len(results) + assert resp.count == NUM_CHUNKS * len(results) + assert rd.minimum >= -0.001 # allow float noise; delivery should be >= 0 + assert resp.minimum >= -0.001 + # loopback delivery should be well under a few ms at p99 + assert rd.p99 < 20.0 + assert resp.p99 < 20.0 + + # the mock keeps its own schedule tightly (its whole job) + assert m[scorer.M_SERVER_TTFC_ABS_ERR].p99 < 15.0 + assert m[scorer.M_SERVER_TPOC_ABS_ERR].p99 < 15.0 + + # client-observed ttfc/tpoc are in the right ballpark + assert m[scorer.M_CLIENT_TTFC].p50 == pytest.approx(TTFC_MS, abs=30.0) + assert m[scorer.M_CLIENT_TPOC].count == (NUM_CHUNKS - 1) * len(results) + assert m[scorer.M_CLIENT_TPOC].p50 == pytest.approx(TPOC_MS, abs=TPOC_MS) diff --git a/tests/unit/preflight/test_driver_integration.py b/tests/unit/preflight/test_driver_integration.py new file mode 100644 index 00000000..8d00073f --- /dev/null +++ b/tests/unit/preflight/test_driver_integration.py @@ -0,0 +1,171 @@ +"""End-to-end: the preflight driver runs the REAL benchmark loop vs a mock. + +Unlike test_chat_mock_integration (which drives the client directly), this +exercises the full scheduler -> dispatch -> client_runner -> client path, so the +dispatch-drift metrics are populated too. Then it gates the result. +""" + +import pytest + +from veeksha.config.preflight import ( + PreflightSttCheckConfig, + PreflightTextCheckConfig, + PreflightTtsCheckConfig, +) +from veeksha.config.traffic import ConcurrentTrafficConfig +from veeksha.preflight import scorer, validator +from veeksha.preflight.drivers import ( + run_completions_preflight, + run_realtime_tts_preflight, + run_stt_preflight, + run_text_preflight, + run_tts_preflight, + run_vajra_tts_preflight, +) + +TTFC_MS = 60.0 +TPOC_MS = 8.0 +NUM_CHUNKS = 16 + + +def _traffic(concurrency): + return ConcurrentTrafficConfig( + target_concurrent_sessions=concurrency, rampup_seconds=0 + ) + + +def _text_cfg(): + return PreflightTextCheckConfig( + input_tokens=8, + num_response_chunks=NUM_CHUNKS, + server_ttfc_ms=TTFC_MS, + server_tpoc_ms=TPOC_MS, + ) + + +def _tts_cfg(): + return PreflightTtsCheckConfig( + input_tokens=8, + input_chunk_tokens=2, + input_pacing_tps=500.0, # fast so the test isn't slow + num_response_chunks=NUM_CHUNKS, + server_ttfc_ms=TTFC_MS, + server_tpoc_ms=TPOC_MS, + ) + + +def _stt_cfg(): + return PreflightSttCheckConfig( + input_seconds=0.3, # short clip keeps the realtime-paced input quick + input_chunk_bytes=1024, + sample_rate=16000, + num_response_chunks=NUM_CHUNKS, + server_ttfc_ms=TTFC_MS, + server_tpoc_ms=TPOC_MS, + ) + + +def _assert_measured(report, streaming_input): + assert report.n_requests > 0 + assert report.n_paired_requests > 0 + assert report.unpaired_fraction < 0.5 + m = report.metrics + assert m[scorer.M_REQUEST_DELIVERY].count > 0 + assert m[scorer.M_RESPONSE_DELIVERY].count > 0 + assert m[scorer.M_SERVER_TTFC_ABS_ERR].count > 0 + # dispatch drift only exists because the real scheduler/dispatch path ran. + assert m[scorer.M_LIFECYCLE_READY_TO_SEND].count > 0 + if streaming_input: + assert m[scorer.M_INPUT_DELIVERY].count > 0 + assert m[scorer.M_INPUT_PACING_ABS_ERR].count > 0 + else: + assert scorer.M_INPUT_DELIVERY not in m + assert scorer.M_INPUT_PACING_ABS_ERR not in m + + +@pytest.mark.unit +def test_text_driver_runs_real_loop_and_scores(tmp_path): + report = run_text_preflight( + _text_cfg(), + traffic_scheduler=_traffic(8), + num_sessions=40, + output_dir=str(tmp_path / "text"), + ) + _assert_measured(report, streaming_input=False) + assert report.metrics[scorer.M_SERVER_TPOC_ABS_ERR].count > 0 + + result = validator.run_validation( + report, + delivery_lag_threshold_ms=25.0, + server_pacing_threshold_ms=25.0, + dispatch_drift_threshold_ms=50.0, + input_pacing_threshold_ms=50.0, + max_unpaired_fraction=0.1, + ) + assert result.verdict in ( + validator.VERDICT_PASS, + validator.VERDICT_FAIL, + validator.VERDICT_SERVER_AT_CAPACITY, + ) + + +@pytest.mark.unit +def test_completions_driver_runs_real_loop_and_scores(tmp_path): + report = run_completions_preflight( + _text_cfg(), + traffic_scheduler=_traffic(8), + num_sessions=40, + output_dir=str(tmp_path / "completions"), + ) + _assert_measured(report, streaming_input=False) + # non-streaming: a single response, so no tpoc samples. + assert scorer.M_SERVER_TPOC_ABS_ERR not in report.metrics + + +@pytest.mark.unit +def test_tts_driver_runs_real_loop_and_scores(tmp_path): + report = run_tts_preflight( + _tts_cfg(), + traffic_scheduler=_traffic(8), + num_sessions=40, + output_dir=str(tmp_path / "tts"), + ) + # HTTP tts sends the whole text in one POST -> no streaming-input metrics. + _assert_measured(report, streaming_input=False) + assert report.metrics[scorer.M_SERVER_TPOC_ABS_ERR].count > 0 + + +@pytest.mark.unit +def test_realtime_tts_driver_runs_real_loop_and_scores(tmp_path): + report = run_realtime_tts_preflight( + _tts_cfg(), + traffic_scheduler=_traffic(6), + num_sessions=24, + output_dir=str(tmp_path / "realtime_tts"), + ) + _assert_measured(report, streaming_input=True) + assert report.metrics[scorer.M_SERVER_TPOC_ABS_ERR].count > 0 + + +@pytest.mark.unit +def test_vajra_tts_driver_runs_real_loop_and_scores(tmp_path): + report = run_vajra_tts_preflight( + _tts_cfg(), + traffic_scheduler=_traffic(6), + num_sessions=24, + output_dir=str(tmp_path / "vajra_tts"), + ) + _assert_measured(report, streaming_input=True) + assert report.metrics[scorer.M_SERVER_TPOC_ABS_ERR].count > 0 + + +@pytest.mark.unit +def test_stt_driver_runs_real_loop_and_scores(tmp_path): + report = run_stt_preflight( + _stt_cfg(), + traffic_scheduler=_traffic(4), + num_sessions=16, + output_dir=str(tmp_path / "stt"), + ) + _assert_measured(report, streaming_input=True) + assert report.metrics[scorer.M_SERVER_TPOC_ABS_ERR].count > 0 diff --git a/tests/unit/preflight/test_prompt_generation.py b/tests/unit/preflight/test_prompt_generation.py new file mode 100644 index 00000000..09650e76 --- /dev/null +++ b/tests/unit/preflight/test_prompt_generation.py @@ -0,0 +1,61 @@ +"""The preflight synthetic text prompt must stream as many paced deltas. + +Regression guard: the preflight tokenizer decodes each token to " " so +PromptStringGenerator tiles whitespace-separated words. Without the boundary +space the prompt collapses to a single space-less blob and segment_text yields +one delta -- which silently disables input-pacing/delivery measurement for the +streaming-text WS clients (realtime_tts, vajra_tts_stream). +""" + +import pytest + +from veeksha.client.utils import segment_text +from veeksha.config.generator.channel import TextChannelGeneratorConfig +from veeksha.config.generator.length import FixedLengthGeneratorConfig +from veeksha.core.seeding import SeedManager +from veeksha.generator.channel.text import TextChannelGenerator +from veeksha.preflight.drivers import _build_preflight_tokenizer_provider +from veeksha.types import ChannelModality + + +def _handle(): + return _build_preflight_tokenizer_provider("preflight-mock").for_modality( + ChannelModality.TEXT + ) + + +def _prompt(input_tokens: int) -> str: + cfg = TextChannelGeneratorConfig( + body_length_generator=FixedLengthGeneratorConfig(value=input_tokens) + ) + gen = TextChannelGenerator(cfg, SeedManager(0), _handle()) + return gen.generate_content(is_root=True).input_text + + +@pytest.mark.unit +def test_decode_gives_per_token_boundary_space(): + handle = _handle() + assert handle.decode([914, 612, 84]) == "914 612 84 " + # The generator's tiling ("".join) must preserve word boundaries. + assert handle.decode([1]) + handle.decode([2]) == "1 2 " + + +@pytest.mark.unit +@pytest.mark.parametrize("ids", [[5], [5, 5], [914, 612, 84], [0, 7, 42]]) +def test_encode_decode_round_trips(ids): + handle = _handle() + assert list(handle.encode(handle.decode(ids))) == ids + + +@pytest.mark.unit +@pytest.mark.parametrize("input_tokens", [16, 100, 512]) +def test_prompt_is_whitespace_delimited_and_multi_segment(input_tokens): + text = _prompt(input_tokens) + # Token count is honored (each token is one whitespace word). + assert len(text.split()) == input_tokens + # And it streams as ~input_tokens/tokens_per_delta deltas, not one blob. + segments = segment_text(text, tokens_per_delta=4) + assert len(segments) == pytest.approx(input_tokens / 4, abs=1) + assert len(segments) > 1 + # segment_text's byte-exact round-trip invariant still holds. + assert "".join(s.text for s in segments) == text diff --git a/tests/unit/preflight/test_scorer.py b/tests/unit/preflight/test_scorer.py new file mode 100644 index 00000000..21cd9d8e --- /dev/null +++ b/tests/unit/preflight/test_scorer.py @@ -0,0 +1,126 @@ +"""Unit tests for the pure drift scorer.""" + +import math +from types import SimpleNamespace + +import pytest + +from veeksha.preflight import scorer +from veeksha.preflight.models import ServerRequestRecord + + +def _result( + request_id, + ready_time, + dispatched_time, + pickup_time, + client_send_time, + client_recv_times, +): + return SimpleNamespace( + request_id=request_id, + scheduler_ready_at=ready_time, + scheduler_dispatched_at=dispatched_time, + client_picked_up_at=pickup_time, + client_sent_at=client_send_time, + chunk_recv_times=client_recv_times, + ) + + +def test_percentile_interpolates(): + assert scorer.percentile([10.0], 99) == 10.0 + assert scorer.percentile([0.0, 10.0], 50) == pytest.approx(5.0) + assert scorer.percentile([0.0, 100.0], 99) == pytest.approx(99.0) + assert math.isnan(scorer.percentile([], 50)) + + +def test_summarize_empty_is_zero_count(): + s = scorer.summarize("x", []) + assert s.count == 0 + assert math.isnan(s.p99) + + +def test_score_computes_all_drifts(): + ttfc_ms, tpoc_ms = 200.0, 20.0 + ready_time = 100.0 + dispatched_time = ready_time + 0.001 # +1ms + pickup_time = dispatched_time + 0.0005 # +0.5ms + client_send_time = pickup_time + 0.0005 # +0.5ms -> ready_to_send = 2ms + server_recv_time = client_send_time + 0.0005 # request delivery 0.5ms + # server emits: ttfc after receipt, then two tpoc gaps -- perfectly on schedule + server_send_times = [ + server_recv_time + 0.200, + server_recv_time + 0.220, + server_recv_time + 0.240, + ] + client_recv_times = [t + 0.0003 for t in server_send_times] # +0.3ms each + + results = [ + _result( + 1, + ready_time, + dispatched_time, + pickup_time, + client_send_time, + client_recv_times, + ) + ] + server_records = { + 1: ServerRequestRecord( + 1, + server_recv_time, + list(server_send_times), + ) + } + + rep = scorer.score(results, server_records, ttfc_ms, tpoc_ms) + m = rep.metrics + + assert rep.n_requests == 1 + assert rep.n_paired_requests == 1 + assert rep.unpaired_fraction == 0.0 + + assert m[scorer.M_LIFECYCLE_READY_TO_DISPATCH].p50 == pytest.approx(1.0, abs=1e-6) + assert m[scorer.M_LIFECYCLE_DISPATCH_TO_PICKUP].p50 == pytest.approx(0.5, abs=1e-6) + assert m[scorer.M_LIFECYCLE_PICKUP_TO_SEND].p50 == pytest.approx(0.5, abs=1e-6) + assert m[scorer.M_LIFECYCLE_READY_TO_SEND].p50 == pytest.approx(2.0, abs=1e-6) + assert m[scorer.M_REQUEST_DELIVERY].p50 == pytest.approx(0.5, abs=1e-6) + assert m[scorer.M_RESPONSE_DELIVERY].count == 3 + assert m[scorer.M_RESPONSE_DELIVERY].p99 == pytest.approx(0.3, abs=1e-6) + assert m[scorer.M_SERVER_TTFC_ABS_ERR].p99 == pytest.approx(0.0, abs=1e-6) + assert m[scorer.M_SERVER_TPOC_ABS_ERR].count == 2 + assert m[scorer.M_SERVER_TPOC_ABS_ERR].p99 == pytest.approx(0.0, abs=1e-6) + assert m[scorer.M_CLIENT_TTFC].p50 == pytest.approx(200.8, abs=1e-3) + # client-observed tpoc: two gaps of ~20ms + assert m[scorer.M_CLIENT_TPOC].count == 2 + assert m[scorer.M_CLIENT_TPOC].p50 == pytest.approx(20.0, abs=1e-3) + + +def test_score_flags_server_pacing_error(): + # Server runs 5ms late on ttfc and 3ms long on every tpoc gap. + ttfc_ms, tpoc_ms = 200.0, 20.0 + server_recv_time = 50.0 + server_send_times = [ + server_recv_time + 0.205, # +5ms ttfc + server_recv_time + 0.228, # +23ms gap + server_recv_time + 0.251, # +23ms gap + ] + r = _result(7, None, None, None, None, None) + rep = scorer.score( + [r], + {7: ServerRequestRecord(7, server_recv_time, list(server_send_times))}, + ttfc_ms, + tpoc_ms, + ) + assert rep.metrics[scorer.M_SERVER_TTFC_ABS_ERR].p50 == pytest.approx(5.0, abs=1e-6) + assert rep.metrics[scorer.M_SERVER_TPOC_ABS_ERR].p99 == pytest.approx(3.0, abs=1e-6) + + +def test_score_unpaired_requests_counted(): + r1 = _result(1, 10.0, 10.001, 10.0015, 10.002, None) + r2 = _result(2, 10.0, 10.001, 10.0015, 10.002, None) + # only request 1 has a server record + rep = scorer.score([r1, r2], {1: ServerRequestRecord(1, 10.0025, [])}, 200.0, 20.0) + assert rep.n_requests == 2 + assert rep.n_paired_requests == 1 + assert rep.unpaired_fraction == pytest.approx(0.5) diff --git a/tests/unit/preflight/test_validator.py b/tests/unit/preflight/test_validator.py new file mode 100644 index 00000000..ad0129bb --- /dev/null +++ b/tests/unit/preflight/test_validator.py @@ -0,0 +1,89 @@ +"""Unit tests for the preflight gate/verdict validator and report renderer.""" + +import pytest + +from veeksha.preflight import scorer, validator +from veeksha.preflight.models import MetricSummary, ScoreReport +from veeksha.preflight.report import render_report + + +def _report(**metric_p99): + """Build a ScoreReport where each given metric has the specified p99.""" + metrics = {} + for name, p99 in metric_p99.items(): + metrics[name] = MetricSummary(name, 100, p99 / 2, p99, p99 / 2, 0.0, p99) + return ScoreReport( + metrics=metrics, n_requests=100, n_paired_requests=100, unpaired_fraction=0.0 + ) + + +_KW = dict( + delivery_lag_threshold_ms=5.0, + server_pacing_threshold_ms=5.0, + dispatch_drift_threshold_ms=10.0, + input_pacing_threshold_ms=10.0, + max_unpaired_fraction=0.02, +) + + +def _all_good(): + return _report( + **{ + scorer.M_SERVER_TTFC_ABS_ERR: 0.5, + scorer.M_SERVER_TPOC_ABS_ERR: 0.5, + scorer.M_REQUEST_DELIVERY: 1.0, + scorer.M_RESPONSE_DELIVERY: 1.0, + scorer.M_LIFECYCLE_READY_TO_SEND: 2.0, + } + ) + + +def test_all_within_threshold_passes(): + result = validator.run_validation(_all_good(), **_KW) + assert result.verdict == validator.VERDICT_PASS + assert result.is_pass + assert all(g.passed for g in result.gates) + + +def test_harness_delivery_breach_fails(): + rep = _all_good() + rep.metrics[scorer.M_RESPONSE_DELIVERY] = MetricSummary( + scorer.M_RESPONSE_DELIVERY, 100, 3.0, 42.0, 5.0, 0.0, 42.0 # p99 42ms >> 5 + ) + result = validator.run_validation(rep, **_KW) + assert result.verdict == validator.VERDICT_FAIL + assert result.failed_gates("harness") + + +def test_server_pacing_breach_is_server_at_capacity(): + # Both a server-pacing gate AND a harness gate fail -> SERVER_AT_CAPACITY wins. + rep = _all_good() + rep.metrics[scorer.M_SERVER_TPOC_ABS_ERR] = MetricSummary( + scorer.M_SERVER_TPOC_ABS_ERR, 100, 4.0, 30.0, 6.0, 0.0, 30.0 + ) + rep.metrics[scorer.M_REQUEST_DELIVERY] = MetricSummary( + scorer.M_REQUEST_DELIVERY, 100, 3.0, 40.0, 5.0, 0.0, 40.0 + ) + result = validator.run_validation(rep, **_KW) + assert result.verdict == validator.VERDICT_SERVER_AT_CAPACITY + + +def test_too_many_unpaired_fails(): + rep = _all_good() + rep.unpaired_fraction = 0.10 # 10% > 2% + result = validator.run_validation(rep, **_KW) + assert result.verdict == validator.VERDICT_FAIL + + +def test_missing_metric_fails_safe(): + # No metrics at all: NaN p99 fails every harness gate -> FAIL. + result = validator.run_validation(_report(), **_KW) + assert result.verdict == validator.VERDICT_FAIL + + +def test_report_renders_verdict_and_gates(): + result = validator.run_validation(_all_good(), **_KW) + text = render_report(_all_good(), result) + assert "VERDICT: PASS" in text + assert "request delivery" in text + assert "PASS" in text diff --git a/veeksha/benchmark.py b/veeksha/benchmark.py index ebe98e6f..ad5355d8 100644 --- a/veeksha/benchmark.py +++ b/veeksha/benchmark.py @@ -36,6 +36,7 @@ logger = init_logger(__name__) +_MAX_NUM_CLIENT_THREADS_DEFAULT = 8 def _warn_if_gil_enabled(stage: str) -> None: """Warn when the GIL is active on a free-threaded build. @@ -111,6 +112,7 @@ def _run_main_loop( num_client_threads = ( max(3, -(-int(target_sessions) // 8)) if target_sessions else 3 ) + num_client_threads = min(_MAX_NUM_CLIENT_THREADS_DEFAULT, num_client_threads) client_queues = [Queue() for _ in range(num_client_threads)] output_queue = Queue() stop_event = threading.Event() diff --git a/veeksha/benchmark_utils.py b/veeksha/benchmark_utils.py index f31147ae..d54e7e85 100644 --- a/veeksha/benchmark_utils.py +++ b/veeksha/benchmark_utils.py @@ -110,8 +110,13 @@ async def warmup_one(session): await client.send_request(first_request, session.id, 1) async def run_all(warmup_sessions): - for session in tqdm(warmup_sessions, desc="Warmup", unit="sess"): - await warmup_one(session) + try: + for session in tqdm(warmup_sessions, desc="Warmup", unit="sess"): + await warmup_one(session) + finally: + # ``asyncio.run`` closes the loop this warmup's session is bound to; + # the benchmark's client threads build their own afterwards. + await client.aclose() if hasattr(session_generator, "get_warmup_sessions"): warmup_sessions = session_generator.get_warmup_sessions() diff --git a/veeksha/cli/commands.py b/veeksha/cli/commands.py index f3dcb0f4..1f10833d 100644 --- a/veeksha/cli/commands.py +++ b/veeksha/cli/commands.py @@ -26,6 +26,7 @@ from veeksha.cli.benchmarks import run_cli as run_benchmark from veeksha.config.benchmark import BenchmarkConfig from veeksha.config.capacity_search import CapacitySearchConfig +from veeksha.config.preflight import PreflightCheckConfig from veeksha.config.score_tts_longform import ScoreTtsLongformConfig from veeksha.microbench.config import ( DecodeMicrobenchmarkConfig, @@ -36,6 +37,7 @@ from veeksha.microbench.diff import DiffConfig, run_diff from veeksha.microbench.prefill import run_prefill from veeksha.microbench.stress import run_stress +from veeksha.preflight.runner import run_preflight_cli from veeksha.verification.longform import run_score_tts_longform_cli from veeksha.version import __version__ @@ -47,6 +49,7 @@ StressMicrobenchmarkConfig: lambda configs: [run_stress(c) for c in configs], DiffConfig: lambda configs: [run_diff(c) for c in configs], ScoreTtsLongformConfig: run_score_tts_longform_cli, + PreflightCheckConfig: run_preflight_cli, } _VERSION_FLAGS = {"--version", "-V"} diff --git a/veeksha/client/base.py b/veeksha/client/base.py index 59d27cf6..77af4939 100644 --- a/veeksha/client/base.py +++ b/veeksha/client/base.py @@ -74,3 +74,10 @@ async def send_request( RequestResult containing response data and timing """ raise NotImplementedError + + async def aclose(self) -> None: + """Release transport state held for the calling thread's event loop. + + No-op by default; HTTP clients override it to close their session. + """ + return None diff --git a/veeksha/client/http_session.py b/veeksha/client/http_session.py new file mode 100644 index 00000000..4b3767ef --- /dev/null +++ b/veeksha/client/http_session.py @@ -0,0 +1,45 @@ +"""Shared aiohttp session construction for the HTTP clients. + +Sessions bind to the event loop running when they are built, so each client +keeps its own thread-local session; this module only holds the transport +configuration they share. +""" + +from __future__ import annotations + +import threading + +import aiohttp + +# aiohttp cannot express "no keepalive expiry" (``keepalive_timeout=None`` +# breaks its pool cleanup arithmetic), so use a value longer than any run. +KEEPALIVE_TIMEOUT_S = 24 * 60 * 60 + + +def new_session(timeout_s: float) -> aiohttp.ClientSession: + """Build a session whose timeouts apply per operation, not per request.""" + return aiohttp.ClientSession( + # ``total`` stays unset so a long generation is not cut off mid-stream; + # ``sock_read`` bounds the gap between chunks. + timeout=aiohttp.ClientTimeout( + total=None, + connect=timeout_s, + sock_connect=timeout_s, + sock_read=timeout_s, + ), + connector=aiohttp.TCPConnector( + # 0 means unlimited; aiohttp otherwise caps at 100 connections, + # which would throttle a high-concurrency benchmark. + limit=0, + limit_per_host=0, + keepalive_timeout=KEEPALIVE_TIMEOUT_S, + ), + ) + + +async def close_session(storage: threading.local) -> None: + """Close this thread's session, if one was ever built on it.""" + session = getattr(storage, "client", None) + if session is not None: + await session.close() + del storage.client diff --git a/veeksha/client/openai_base.py b/veeksha/client/openai_base.py index 1f0a7f0d..503ff4e5 100644 --- a/veeksha/client/openai_base.py +++ b/veeksha/client/openai_base.py @@ -6,9 +6,10 @@ import threading from typing import Any, Dict -import httpx +import aiohttp from veeksha.client.base import BaseLLMClient +from veeksha.client.http_session import close_session, new_session from veeksha.core.request import Request from veeksha.core.tokenizer import TokenizerProvider from veeksha.types import ChannelModality @@ -18,7 +19,7 @@ class OpenAIBaseClient(BaseLLMClient): """Common base for OpenAI-compatible clients. This includes: - - Thread-local httpx.AsyncClient management + - Thread-local aiohttp.ClientSession management - Token counting helpers - Global + per-request sampling params merge logic """ @@ -41,14 +42,16 @@ def _get_cached_token_count(self, text: str) -> int: """Return token count for text with caching.""" return len(self.text_tokenizer_handle.encode(text)) - def _get_client(self) -> httpx.AsyncClient: - """Get or create a thread-local httpx client.""" + def _get_client(self) -> aiohttp.ClientSession: + """Get or create a thread-local aiohttp session.""" if not hasattr(self.client_storage, "client"): - self.client_storage.client = httpx.AsyncClient( - timeout=self.config.request_timeout - ) + self.client_storage.client = new_session(self.config.request_timeout) return self.client_storage.client + async def aclose(self) -> None: + """Close the session bound to the calling thread's event loop.""" + await close_session(self.client_storage) + def _get_sampling_params(self, request: Request) -> Dict[str, Any]: """Merge global and per-request sampling params. diff --git a/veeksha/client/openai_chat.py b/veeksha/client/openai_chat.py index d124e483..d2af67c7 100644 --- a/veeksha/client/openai_chat.py +++ b/veeksha/client/openai_chat.py @@ -1,10 +1,11 @@ from __future__ import annotations +import asyncio import base64 import time from typing import TYPE_CHECKING, Any, Callable, List, Optional -import httpx # type: ignore +import aiohttp from veeksha.client.openai_base import OpenAIBaseClient from veeksha.core.audio_contract import DEFAULT_AUDIO_SAMPLE_RATE, AudioMetricKey @@ -27,7 +28,7 @@ class OpenAIChatCompletionsClient(OpenAIBaseClient): - """Async client for OpenAI Chat Completions API using httpx. + """Async client for OpenAI Chat Completions API using aiohttp. Works with new Request objects that have channels instead of prompt tuples. """ @@ -291,21 +292,21 @@ def _build_channel_responses( return channels - async def _process_stream(self, response: httpx.Response): + async def _process_stream(self, response: aiohttp.ClientResponse): """Process SSE stream from server.""" import json - buffer = "" - async for chunk in response.aiter_text(): + buffer = b"" + async for chunk in response.content.iter_any(): buffer += chunk - while "\n" in buffer: - line, buffer = buffer.split("\n", 1) + while b"\n" in buffer: + line, buffer = buffer.split(b"\n", 1) line = line.strip() if not line: continue - if line.startswith("data:"): + if line.startswith(b"data:"): data_str = line[5:].strip() - if data_str == "[DONE]": + if data_str == b"[DONE]": return try: yield json.loads(data_str) @@ -322,8 +323,6 @@ async def send_request( ) -> RequestResult: """Send a streaming request to the OpenAI Chat Completions API.""" - timeout = self.config.request_timeout - max_tokens_limit = None if ( request.requested_output is not None @@ -338,6 +337,12 @@ async def send_request( chunks_received = 0 generated_text = "" + # preflight timing (recorded only when enabled). Only the request id is + # sent to the server; the scorer joins the two record books by request_id. + preflight_enabled = getattr(self.config, "record_preflight_timing", False) + client_sent_at: Optional[float] = None + chunk_recv_times: List[float] = [] + # multimodal response data image_data: Optional[Any] = None video_data: Optional[Any] = None @@ -391,12 +396,14 @@ async def send_request( client = self._get_client() t_start = time.monotonic() most_recent_token_time = t_start - async with client.stream( - "POST", + # t_cs: request handed to the transport (always recorded). + client_sent_at = t_start + if preflight_enabled: + headers["X-Veeksha-Request-Id"] = str(request.id) + async with client.post( self.chat_address, json=body, headers=headers, - timeout=timeout, ) as response: response.raise_for_status() @@ -406,6 +413,9 @@ async def send_request( sent_notified = False async for data in self._process_stream(response): receive_time = time.monotonic() + # t_cr_i: client receipt of each response chunk. + if preflight_enabled: + chunk_recv_times.append(receive_time) if "error" in data: err = data.get("error") or {} error_msg = err.get("message", "Unknown error") @@ -459,18 +469,18 @@ async def send_request( # TODO: video deltas video_data = self._process_video_response(delta, video_data) - except httpx.HTTPStatusError as e: - error_code = e.response.status_code if e.response else 500 + except aiohttp.ClientResponseError as e: + error_code = e.status or 500 error_msg = error_msg or str(e) logger.warning(f"HTTP Error: status={error_code} msg={error_msg}") - except httpx.ConnectError as e: - error_code = 503 - error_msg = error_msg or str(e) - logger.warning(f"Connection Error: ({error_code}) {error_msg}") - except httpx.TimeoutException: + except asyncio.TimeoutError: error_code = 408 error_msg = error_msg or "Request timed out" logger.warning(f"Timeout Error: ({error_code}) {error_msg}") + except aiohttp.ClientConnectorError as e: + error_code = 503 + error_msg = error_msg or str(e) + logger.warning(f"Connection Error: ({error_code}) {error_msg}") except Exception as e: error_code = error_code or 520 error_msg = error_msg or str(e) @@ -531,4 +541,6 @@ async def send_request( error_code=error_code, error_msg=error_msg, client_completed_at=completed_at, + client_sent_at=client_sent_at, + chunk_recv_times=chunk_recv_times if preflight_enabled else None, ) diff --git a/veeksha/client/openai_completions.py b/veeksha/client/openai_completions.py index 1232549e..f62ac039 100644 --- a/veeksha/client/openai_completions.py +++ b/veeksha/client/openai_completions.py @@ -2,10 +2,11 @@ from __future__ import annotations +import asyncio import time from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional -import httpx +import aiohttp from veeksha.client.openai_base import OpenAIBaseClient from veeksha.core.request import Request @@ -21,7 +22,7 @@ class OpenAICompletionsClient(OpenAIBaseClient): - """Async client for OpenAI `/completions` API using httpx. + """Async client for OpenAI `/completions` API using aiohttp. This client always uses the completions endpoint (non-streaming). It is intended for servers that only expose completions, or workloads that require @@ -68,8 +69,6 @@ async def _send_completions_request( on_request_dispatched: Optional[Callable[[], None]] = None, ) -> RequestResult: """Execute the HTTP request against `/completions` and parse the response.""" - timeout = self.config.request_timeout - prompt_text = "" if ChannelModality.TEXT in request.channels: text_content = request.channels[ChannelModality.TEXT] @@ -118,35 +117,50 @@ async def _send_completions_request( completion_text = "" logprobs: Any = None + # preflight timing (recorded only when enabled). Only the request id is + # sent to the server; the scorer joins the two record books by request_id. + # Completions is non-streaming: a single t_cr. + preflight_enabled = getattr(self.config, "record_preflight_timing", False) + client_sent_at: Optional[float] = None + chunk_recv_times: List[float] = [] + start_time = time.monotonic() + client_sent_at = start_time + if preflight_enabled: + headers["X-Veeksha-Request-Id"] = str(request.id) try: client = self._get_client() - response = await client.post( - self.completions_address, json=body, headers=headers, timeout=timeout - ) - response.raise_for_status() - if on_request_dispatched is not None: - on_request_dispatched() - if on_request_sent is not None: - on_request_sent() - data = response.json() + async with client.post( + self.completions_address, json=body, headers=headers + ) as response: + response.raise_for_status() + if on_request_dispatched is not None: + on_request_dispatched() + if on_request_sent is not None: + on_request_sent() + # aiohttp returns on headers, so t_cr (response fully received) + # is only true once the body has been read. + data = await response.json(content_type=None) + receive_time = time.monotonic() + if preflight_enabled: + chunk_recv_times.append(receive_time) choices = data.get("choices") or [] if choices: first = choices[0] if isinstance(choices[0], dict) else {} generated_text = first.get("text", "") or "" logprobs = first.get("logprobs") - except httpx.HTTPStatusError as e: - error_code = e.response.status_code if e.response else 500 + except aiohttp.ClientResponseError as e: + error_code = e.status or 500 error_msg = error_msg or str(e) logger.warning("HTTP Error: status=%s msg=%s", error_code, error_msg) - except httpx.ConnectError as e: - error_code = 503 - error_msg = error_msg or str(e) - logger.warning("Connection Error: (%s) %s", error_code, error_msg) - except httpx.TimeoutException: + except asyncio.TimeoutError: error_code = 408 error_msg = error_msg or "Request timed out" logger.warning("Timeout Error: (%s) %s", error_code, error_msg) + except aiohttp.ClientConnectorError as e: + error_code = 503 + error_msg = error_msg or str(e) + logger.warning("Connection Error: (%s) %s", error_code, error_msg) except Exception as e: error_code = error_code or 520 error_msg = error_msg or str(e) @@ -203,4 +217,6 @@ async def _send_completions_request( error_code=error_code, error_msg=error_msg, client_completed_at=completed_at, + client_sent_at=client_sent_at, + chunk_recv_times=chunk_recv_times if preflight_enabled else None, ) diff --git a/veeksha/client/openai_router.py b/veeksha/client/openai_router.py index b1ee78c9..453538d6 100644 --- a/veeksha/client/openai_router.py +++ b/veeksha/client/openai_router.py @@ -75,3 +75,8 @@ async def send_request( on_request_sent=on_request_sent, on_request_dispatched=on_request_dispatched, ) + + async def aclose(self) -> None: + """Close both sub-clients; each holds its own session.""" + await self._chat_client.aclose() + await self._completions_client.aclose() diff --git a/veeksha/client/realtime_tts.py b/veeksha/client/realtime_tts.py index 0860ebae..c27a037a 100644 --- a/veeksha/client/realtime_tts.py +++ b/veeksha/client/realtime_tts.py @@ -196,21 +196,25 @@ def __init__(self, config: "RealtimeTTSClientConfig", **kwargs) -> None: self._realtime_config = config self._protocol = _build_realtime_protocol(config, api_key=self.api_key) - def _connect(self): + def _connect(self, extra_headers: Optional[dict] = None): """Return the websocket connect context manager. A seam so tests can override the transport. ``max_size=None`` lifts the inbound-frame cap (audio deltas can be large); ``compression=None`` keeps binary PCM uncompressed. The asyncio transport sets ``TCP_NODELAY`` - by default, so no explicit socket option is required. + by default, so no explicit socket option is required. ``extra_headers`` + (preflight only) adds the request-id correlation header. """ open_timeout = min(self._realtime_config.request_timeout, 30) + headers = self._protocol.headers() + if extra_headers: + headers = {**headers, **extra_headers} return connect( self._protocol.build_ws_url(str(self.api_base)), max_size=None, compression=None, open_timeout=open_timeout, - additional_headers=self._protocol.headers(), + additional_headers=headers, ) async def send_request( @@ -274,7 +278,16 @@ def fire_sent_once() -> None: error_code: Optional[int] = None error_msg: Optional[str] = None + # preflight timing (recorded only when enabled). Only the request id is + # sent to the server; the scorer joins the two record books by request_id. + preflight_enabled = getattr(self.config, "record_preflight_timing", False) + client_sent_at: Optional[float] = None + chunk_recv_times: list[float] = [] + input_send_times: list[float] = [] # t_cs_i, per paced text segment + input_send_deadlines: list[float] = [] # intended send instant per segment + t_start = time.monotonic() + client_sent_at = t_start async def send_loop(ws) -> None: nonlocal input_complete_offset @@ -289,6 +302,10 @@ async def send_loop(ws) -> None: if sleep_s > 0: await asyncio.sleep(sleep_s) await ws.send(self._protocol.conversation_item_create_json(seg.text)) + if preflight_enabled: + # t_cs_i (actual send) vs deadline (intended) -> pacing error. + input_send_times.append(time.monotonic()) + input_send_deadlines.append(deadline) text_delta_ts.append([(time.monotonic() - t_start) * 1000, seg.n_chars]) input_complete_offset = (time.monotonic() - t_start) * 1000 await ws.send(self._protocol.response_create_json()) @@ -298,8 +315,9 @@ async def recv_loop(ws) -> None: nonlocal audio_done_offset, response_done_offset, sample_rate while True: raw = await ws.recv() - # Stamp receipt BEFORE any json/base64 decode work. - offset_ms = (time.monotonic() - t_start) * 1000 + # Stamp receipt BEFORE any json/base64 decode work (t_cr_i). + recv_time = time.monotonic() + offset_ms = (recv_time - t_start) * 1000 try: event = json.loads(raw) except (json.JSONDecodeError, TypeError, ValueError): @@ -313,6 +331,8 @@ async def recv_loop(ws) -> None: if chunk: audio_chunks.append(chunk) audio_chunk_ts.append([offset_ms, len(chunk)]) + if preflight_enabled: + chunk_recv_times.append(recv_time) if ttfc is None: ttfc = offset_ms fire_sent_once() @@ -348,9 +368,12 @@ async def recv_loop(ws) -> None: raise RealtimeServerError(event) # OTHER: keep receiving until response.done. + extra_headers = ( + {"X-Veeksha-Request-Id": str(request.id)} if preflight_enabled else None + ) try: async with asyncio.timeout(self._realtime_config.request_timeout): - async with self._connect() as ws: + async with self._connect(extra_headers) as ws: ws_connect_latency = (time.monotonic() - t_start) * 1000 await ws.send(self._protocol.session_update_json()) # Analog of the HTTP-200 ack: the scheduler's dispatch pacing @@ -431,6 +454,10 @@ async def recv_loop(ws) -> None: error_code=error_code, error_msg=error_msg, client_completed_at=completed_at, + client_sent_at=client_sent_at, + chunk_recv_times=chunk_recv_times if preflight_enabled else None, + input_send_times=input_send_times if preflight_enabled else None, + input_send_deadlines=input_send_deadlines if preflight_enabled else None, ) diff --git a/veeksha/client/stt.py b/veeksha/client/stt.py index 4d33ff25..75124f31 100644 --- a/veeksha/client/stt.py +++ b/veeksha/client/stt.py @@ -169,6 +169,11 @@ class STTStreamResult: transcript_snapshots: list[TranscriptSnapshotRow] chunk_count: int pcm_byte_count: int + # preflight timing (client record book, populated only when enabled) + client_sent_at: Optional[float] = None + chunk_recv_times: Optional[list[float]] = None + input_send_times: Optional[list[float]] = None + input_send_deadlines: Optional[list[float]] = None class TranscriptSnapshotRecorder: @@ -323,6 +328,7 @@ async def _stream( wire_messages: Optional[list[str | bytes]] = None, on_request_sent: Optional[Callable[[], None]] = None, on_request_dispatched: Optional[Callable[[], None]] = None, + request_id: Optional[int] = None, ) -> STTStreamResult: """Stream pre-decoded PCM16 to the provider and collect the transcript. @@ -349,11 +355,28 @@ async def _stream( transcript_chunks: list[str] = [] snapshots = TranscriptSnapshotRecorder() + # t_cs: request handed to the transport (always recorded, so the harness + # lifecycle dispatch-drift metrics cover STT on normal runs too). + client_sent_at = time.monotonic() + + # preflight timing (recorded only when enabled). Only the request id is + # sent to the server; the scorer joins the two record books by request_id. + preflight_enabled = getattr(self.config, "record_preflight_timing", False) + chunk_recv_times: list[float] = [] + input_send_times: list[float] = [] # t_cs_i, per paced audio chunk + input_send_deadlines: list[float] = [] + extra_headers = ( + {"X-Veeksha-Request-Id": str(request_id)} + if preflight_enabled and request_id is not None + else None + ) + async with websockets.connect( self._ws_url, ping_interval=self._ws_ping_interval_s, ping_timeout=self._ws_ping_timeout_s, compression=self._ws_compression, + additional_headers=extra_headers, ) as ws: await self._open_session(ws) if on_request_dispatched is not None: @@ -376,6 +399,13 @@ async def _send() -> None: else: message = self._encode_chunk(pcm_bytes[byte_offset:chunk_end]) await ws.send(message) + if preflight_enabled and audio_started_at is not None: + # t_cs_i vs the audio-clock deadline this chunk paces to. + input_send_times.append(time.monotonic()) + input_send_deadlines.append( + audio_started_at + + byte_offset / BYTES_PER_SAMPLE / self._sample_rate + ) if audio_started_at is not None: await self._maybe_pace_until( audio_started_at @@ -388,8 +418,11 @@ async def _send() -> None: try: while True: kind, text = self._parse_message(json.loads(await ws.recv())) - now = time.monotonic() + now = time.monotonic() # t_cr_i + if kind == "delta": + if preflight_enabled: + chunk_recv_times.append(now) # TTFC counts only deltas whose own payload carries # transcript text after cleaning; empty progress / # keepalive deltas (e.g. Vajra's priming delta) and @@ -475,6 +508,10 @@ async def _send() -> None: transcript_snapshots=snapshots.snapshots, chunk_count=chunk_count, pcm_byte_count=len(pcm_bytes), + client_sent_at=client_sent_at, + chunk_recv_times=chunk_recv_times if preflight_enabled else None, + input_send_times=input_send_times if preflight_enabled else None, + input_send_deadlines=input_send_deadlines if preflight_enabled else None, ) # ------------------------------------------------------------------ @@ -596,6 +633,7 @@ def finish_callbacks() -> None: wire_messages, on_request_sent=fire_sent_once, on_request_dispatched=fire_dispatched_once, + request_id=request.id, ) except TimeoutError: error_code = 408 @@ -676,6 +714,20 @@ def finish_callbacks() -> None: error_code=error_code, error_msg=error_msg, client_completed_at=completed_at, + client_sent_at=( + stream_result.client_sent_at if stream_result is not None else None + ), + chunk_recv_times=( + stream_result.chunk_recv_times if stream_result is not None else None + ), + input_send_times=( + stream_result.input_send_times if stream_result is not None else None + ), + input_send_deadlines=( + stream_result.input_send_deadlines + if stream_result is not None + else None + ), ) diff --git a/veeksha/client/tts.py b/veeksha/client/tts.py index ed8ea17e..76225709 100644 --- a/veeksha/client/tts.py +++ b/veeksha/client/tts.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import threading import time from collections.abc import Callable @@ -9,9 +10,10 @@ from typing import TYPE_CHECKING from urllib.parse import urljoin -import httpx +import aiohttp from veeksha.client.base import BaseLLMClient +from veeksha.client.http_session import close_session, new_session from veeksha.core.audio_contract import AudioMetricKey from veeksha.core.request import Request from veeksha.core.request_content import TextChannelRequestContent @@ -73,7 +75,7 @@ def _build_request(self, text: str) -> OpenAISpeechRequest: }, ) - def _validate_audio_response(self, response: httpx.Response) -> None: + def _validate_audio_response(self, response: aiohttp.ClientResponse) -> None: content_type = response.headers.get("Content-Type", "") media_type = content_type.partition(";")[0].strip().lower() allowed = { @@ -91,14 +93,16 @@ def _validate_audio_response(self, response: httpx.Response) -> None: f"response, got Content-Type {content_type!r}" ) - def _get_client(self) -> httpx.AsyncClient: - """Return a thread-local httpx client bound to the caller's event loop.""" + def _get_client(self) -> aiohttp.ClientSession: + """Return a thread-local aiohttp session bound to the caller's event loop.""" if not hasattr(self._client_storage, "client"): - self._client_storage.client = httpx.AsyncClient( - timeout=self.config.request_timeout - ) + self._client_storage.client = new_session(self.config.request_timeout) return self._client_storage.client + async def aclose(self) -> None: + """Close the session bound to the calling thread's event loop.""" + await close_session(self._client_storage) + async def send_request( self, request: Request, @@ -136,15 +140,22 @@ async def send_request( chunk_count = 0 audio_chunks: list[bytes] = [] + # preflight timing (recorded only when enabled). Only the request id is + # sent to the server; the scorer joins the two record books by request_id. + preflight_enabled = getattr(self.config, "record_preflight_timing", False) + client_sent_at: float | None = None + chunk_recv_times: list[float] = [] + if preflight_enabled: + speech_request.headers["X-Veeksha-Request-Id"] = str(request.id) + t_start = time.monotonic() + client_sent_at = t_start try: - async with self._get_client().stream( - "POST", + async with self._get_client().post( speech_request.url, headers=speech_request.headers, json=speech_request.payload, - timeout=self.config.request_timeout, ) as response: response.raise_for_status() self._validate_audio_response(response) @@ -152,10 +163,13 @@ async def send_request( on_request_dispatched() sent_notified = False - async for chunk in response.aiter_bytes(chunk_size=self._chunk_size): + async for chunk in response.content.iter_chunked(self._chunk_size): if not chunk: continue receive_time = time.monotonic() + # t_cr_i: client receipt of each audio chunk. + if preflight_enabled: + chunk_recv_times.append(receive_time) if ttfc is None: ttfc = (receive_time - t_start) * 1000 if not sent_notified and on_request_sent is not None: @@ -172,18 +186,18 @@ async def send_request( error_code = 502 error_msg = str(e) logger.warning("TTS protocol error: (%s) %s", error_code, error_msg) - except httpx.HTTPStatusError as e: - error_code = e.response.status_code if e.response else 500 + except aiohttp.ClientResponseError as e: + error_code = e.status or 500 error_msg = str(e) logger.warning("HTTP Error: status=%s msg=%s", error_code, error_msg) - except httpx.ConnectError as e: - error_code = 503 - error_msg = str(e) - logger.warning("Connection Error: (%s) %s", error_code, error_msg) - except httpx.TimeoutException: + except asyncio.TimeoutError: error_code = 408 error_msg = "TTS request timed out" logger.warning("Timeout Error: (%s) %s", error_code, error_msg) + except aiohttp.ClientConnectorError as e: + error_code = 503 + error_msg = str(e) + logger.warning("Connection Error: (%s) %s", error_code, error_msg) except Exception as e: error_code = 520 error_msg = str(e) @@ -222,4 +236,6 @@ async def send_request( error_code=error_code, error_msg=error_msg, client_completed_at=completed_at, + client_sent_at=client_sent_at, + chunk_recv_times=chunk_recv_times if preflight_enabled else None, ) diff --git a/veeksha/client/vajra_tts_stream.py b/veeksha/client/vajra_tts_stream.py index f42149e0..863633ce 100644 --- a/veeksha/client/vajra_tts_stream.py +++ b/veeksha/client/vajra_tts_stream.py @@ -176,22 +176,26 @@ def __init__(self, config: "VajraTTSStreamClientConfig", **kwargs) -> None: self._stream_config = config self._protocol = VajraTTSStreamProtocol(config, api_key=self.api_key) - def _connect(self): + def _connect(self, extra_headers: Optional[dict] = None): """Return the websocket connect context manager. A seam so tests can override the transport. ``max_size=None`` lifts the inbound-frame cap (PCM frames can be large); ``compression=None`` keeps binary PCM uncompressed (permessage-deflate burns CPU on high-entropy audio). The asyncio transport sets ``TCP_NODELAY`` by - default, so no explicit socket option is required. + default, so no explicit socket option is required. ``extra_headers`` + (preflight only) adds the request-id correlation header. """ open_timeout = min(self._stream_config.request_timeout, 30) + headers = self._protocol.headers() + if extra_headers: + headers = {**headers, **extra_headers} return connect( self._protocol.build_ws_url(str(self.api_base)), max_size=None, compression=None, open_timeout=open_timeout, - additional_headers=self._protocol.headers(), + additional_headers=headers, ) async def send_request( @@ -272,7 +276,16 @@ def fire_sent_once() -> None: error_code: Optional[int] = None error_msg: Optional[str] = None + # preflight timing (recorded only when enabled). Only the request id is + # sent to the server; the scorer joins the two record books by request_id. + preflight_enabled = getattr(self.config, "record_preflight_timing", False) + client_sent_at: Optional[float] = None + chunk_recv_times: list[float] = [] + input_send_times: list[float] = [] # t_cs_i, per paced text segment + input_send_deadlines: list[float] = [] + t_start = time.monotonic() + client_sent_at = t_start async def send_loop(ws) -> None: nonlocal input_complete_offset @@ -287,6 +300,9 @@ async def send_loop(ws) -> None: if sleep_s > 0: await asyncio.sleep(sleep_s) await ws.send(self._protocol.input_text_json(seg.text)) + if preflight_enabled: + input_send_times.append(time.monotonic()) + input_send_deadlines.append(deadline) text_delta_ts.append([(time.monotonic() - t_start) * 1000, seg.n_chars]) # Abort after a fraction of the input deltas: the client stops # feeding text and never sends input.done (hangs up mid-input). @@ -301,8 +317,9 @@ async def recv_loop(ws) -> None: received_audio_ms = 0.0 while True: raw = await ws.recv() - # Stamp receipt BEFORE any json decode work. - offset_ms = (time.monotonic() - t_start) * 1000 + # Stamp receipt BEFORE any json decode work (t_cr_i). + recv_time = time.monotonic() + offset_ms = (recv_time - t_start) * 1000 # Binary frames are raw int16 PCM audio. if isinstance(raw, (bytes, bytearray, memoryview)): @@ -310,6 +327,8 @@ async def recv_loop(ws) -> None: if chunk: audio_chunks.append(chunk) audio_chunk_ts.append([offset_ms, len(chunk)]) + if preflight_enabled: + chunk_recv_times.append(recv_time) if ttfc is None: ttfc = offset_ms fire_sent_once() @@ -366,9 +385,12 @@ async def abort_watchdog() -> None: await asyncio.sleep(abort_wall_s) raise _ClientAbort() + extra_headers = ( + {"X-Veeksha-Request-Id": str(request.id)} if preflight_enabled else None + ) try: async with asyncio.timeout(self._stream_config.request_timeout): - async with self._connect() as ws: + async with self._connect(extra_headers) as ws: ws_connect_latency = (time.monotonic() - t_start) * 1000 await ws.send(self._protocol.session_config_json()) # Analog of the HTTP-200 ack: the scheduler's dispatch pacing @@ -462,4 +484,8 @@ async def abort_watchdog() -> None: error_code=error_code, error_msg=error_msg, client_completed_at=completed_at, + client_sent_at=client_sent_at, + chunk_recv_times=chunk_recv_times if preflight_enabled else None, + input_send_times=input_send_times if preflight_enabled else None, + input_send_deadlines=input_send_deadlines if preflight_enabled else None, ) diff --git a/veeksha/config/client.py b/veeksha/config/client.py index 3381c18e..b877fc0b 100644 --- a/veeksha/config/client.py +++ b/veeksha/config/client.py @@ -35,6 +35,15 @@ class BaseClientConfig(BasePolyConfig): "{}", help="Additional sampling params to send with each request to the LLM API.", ) + record_preflight_timing: bool = field( + False, + help=( + "Preflight only. When set, the client stamps its send time on each " + "request (X-Veeksha-* headers) and records per-chunk send/receive " + "timestamps onto the RequestResult so the preflight scorer can " + "measure request/response delivery drift. No effect on normal runs." + ), + ) def __post_init__(self): self.additional_sampling_params_dict = {} diff --git a/veeksha/config/evaluator.py b/veeksha/config/evaluator.py index 999dd1f4..69fe498c 100644 --- a/veeksha/config/evaluator.py +++ b/veeksha/config/evaluator.py @@ -236,6 +236,16 @@ class PerformanceEvaluatorConfig(BaseEvaluatorConfig): video_channel: Optional[VideoChannelPerformanceConfig] = field( None, help="Video channel performance configuration" ) + lifecycle_drift_warn_threshold_ms: float = field( + 50.0, + help=( + "Soft-warn threshold. If the p99 of any request-lifecycle drift " + "(scheduler-ready -> dispatched -> picked-up -> sent) exceeds this, a " + "warning is logged at the end of the run: the harness itself is " + "adding scheduling latency that can distort measured server timings. " + "Set <= 0 to disable the warning." + ), + ) @classmethod def get_type(cls) -> EvaluationType: diff --git a/veeksha/config/generator/channel.py b/veeksha/config/generator/channel.py index 3064f648..5784785c 100644 --- a/veeksha/config/generator/channel.py +++ b/veeksha/config/generator/channel.py @@ -48,8 +48,29 @@ def get_type(cls): @frozen_dataclass class AudioChannelGeneratorConfig(BaseChannelGeneratorConfig): + """Synthetic audio input: a deterministic generated waveform clip. + + Writes a WAV (cached per parameter set) and hands its path to the client as + audio input. Content is synthetic (a tone or silence) -- useful for STT + smoke-testing and preflight without a real audio dataset. + """ + + duration_seconds: float = field( + 3.0, help="Length of the generated audio clip in seconds." + ) + sample_rate: int = field(16000, help="Audio sample rate in Hz.") + waveform: str = field("sine", help="Waveform to synthesize: 'sine' or 'silence'.") + frequency_hz: float = field( + 440.0, help="Tone frequency in Hz (used when waveform='sine')." + ) + def __post_init__(self): - raise NotImplementedError("AudioChannelConfig is not implemented") + if self.duration_seconds <= 0: + raise ValueError("duration_seconds must be positive") + if self.sample_rate <= 0: + raise ValueError("sample_rate must be positive") + if self.waveform not in ("sine", "silence"): + raise ValueError("waveform must be 'sine' or 'silence'") @classmethod def get_type(cls): diff --git a/veeksha/config/preflight.py b/veeksha/config/preflight.py new file mode 100644 index 00000000..076b43d8 --- /dev/null +++ b/veeksha/config/preflight.py @@ -0,0 +1,185 @@ +"""Configuration for the ``veeksha preflight`` command. + +Preflight certifies that the benchmark harness keeps time honestly before you +trust its numbers: it drives the real scheduler -> dispatch -> client path +against a deterministic mock server and gates the p99 timing drift (request / +response delivery, server pacing fidelity, dispatch drift) against thresholds. + +Workload/timing is grouped by client category -- ``text`` (chat, completions), +``tts`` (tts, realtime_tts, vajra_tts_stream), and ``stt`` -- because the +response shape differs (token stream vs audio chunks vs transcript deltas). + +Traffic is closed-loop concurrency: ``concurrency`` sessions in flight out of +``num_sessions`` total, built with ``rampup_seconds=0`` so measurement runs at +steady state. +""" + +from vidhi import field, frozen_dataclass + +from veeksha.cli.base import VeekshaCommand +from veeksha.config.traffic import BaseTrafficConfig, ConcurrentTrafficConfig + + +@frozen_dataclass +class PreflightTextCheckConfig: + """Workload + mock timing for the text checks (chat, completions). + + Input is a single-shot text prompt; the mock streams a token response. + """ + + input_tokens: int = field(100, help="Prompt length in tokens.") + num_response_chunks: int = field( + 100, help="Response tokens the mock emits (completions emits one response)." + ) + server_ttfc_ms: float = field(200.0, help="Mock time-to-first-chunk delay (ms).") + server_tpoc_ms: float = field( + 20.0, help="Mock time-per-output-chunk (inter-chunk) delay (ms)." + ) + + +@frozen_dataclass +class PreflightTtsCheckConfig: + """Workload + mock timing for the TTS checks (tts, realtime_tts, vajra). + + Input is a text prompt (streamed in paced segments for the WebSocket + clients); the mock streams an audio-chunk response. + """ + + input_tokens: int = field(100, help="Prompt length in tokens.") + input_chunk_tokens: int = field( + 4, help="Tokens per streamed input message (realtime_tts / vajra)." + ) + input_pacing_tps: float = field( + 50.0, help="Input pacing rate in tokens/sec (realtime_tts / vajra)." + ) + num_response_chunks: int = field(100, help="Audio chunks the mock emits.") + server_ttfc_ms: float = field(200.0, help="Mock time-to-first-chunk delay (ms).") + server_tpoc_ms: float = field(20.0, help="Mock inter-audio-chunk delay (ms).") + + +@frozen_dataclass +class PreflightSttCheckConfig: + """Workload + mock timing for the STT check. + + Input is streamed audio; the mock streams a transcript-delta response. + """ + + input_seconds: float = field(3.0, help="Generated audio clip length in seconds.") + input_chunk_bytes: int = field(4096, help="Audio bytes per streamed input message.") + sample_rate: int = field(16000, help="Audio sample rate in Hz.") + num_response_chunks: int = field(40, help="Transcript deltas the mock emits.") + server_ttfc_ms: float = field(200.0, help="Mock time-to-first-chunk delay (ms).") + server_tpoc_ms: float = field(50.0, help="Mock inter-transcript-delta delay (ms).") + + +@frozen_dataclass +class PreflightCheckConfig(VeekshaCommand, name="preflight"): + """Certify measurement fidelity of the harness at a target concurrency.""" + + # --- which client pathways to exercise --- + check_text: bool = field( + True, aliases=["check-text"], help="Run the OpenAI chat check." + ) + check_completions: bool = field( + True, aliases=["check-completions"], help="Run the completions check." + ) + check_tts: bool = field( + True, aliases=["check-tts"], help="Run the TTS (HTTP streaming audio) check." + ) + check_realtime_tts: bool = field( + True, aliases=["check-realtime-tts"], help="Run the realtime-TTS (WS) check." + ) + check_vajra_tts: bool = field( + True, aliases=["check-vajra-tts"], help="Run the Vajra TTS-stream (WS) check." + ) + check_stt: bool = field( + True, aliases=["check-stt"], help="Run the STT (WS audio-in) check." + ) + + # --- traffic (closed-loop concurrency; scheduler built with rampup=0) --- + concurrency: int = field( + 50, + help="Target number of concurrent sessions to sustain (the load level).", + ) + num_sessions: int = field( + 500, + aliases=["num-sessions"], + help="Total sessions to drive across the run (the sample size).", + ) + + # --- per-category workload + mock timing --- + text: PreflightTextCheckConfig = field(default_factory=PreflightTextCheckConfig) + tts: PreflightTtsCheckConfig = field(default_factory=PreflightTtsCheckConfig) + stt: PreflightSttCheckConfig = field(default_factory=PreflightSttCheckConfig) + + # --- gate thresholds (p99, in milliseconds unless noted) --- + delivery_lag_threshold_ms: float = field( + 5.0, + aliases=["delivery-lag-threshold-ms"], + help=( + "Max allowed p99 for request/response/input delivery lag (client<->" + "server). Above this the harness is adding transport/queueing drift." + ), + ) + server_pacing_threshold_ms: float = field( + 5.0, + aliases=["server-pacing-threshold-ms"], + help=( + "Max allowed p99 for the mock server's own ttfc/tpoc pacing error. " + "If exceeded the server itself is the bottleneck (SERVER_AT_CAPACITY)." + ), + ) + dispatch_drift_threshold_ms: float = field( + 10.0, + aliases=["dispatch-drift-threshold-ms"], + help=( + "Max allowed p99 for end-to-end dispatch drift (client-sent minus " + "scheduled-ready). Above this the harness dispatches off-schedule." + ), + ) + input_pacing_threshold_ms: float = field( + 10.0, + aliases=["input-pacing-threshold-ms"], + help=( + "Streaming-input clients only. Max allowed p99 for input pacing " + "error (actual segment send time minus its intended deadline)." + ), + ) + max_unpaired_fraction: float = field( + 0.02, + aliases=["max-unpaired-fraction"], + help="Max fraction of requests allowed to lack a matching server record.", + ) + + # --- output --- + output_dir: str = field( + "preflight_report", + aliases=["output-dir"], + help="Directory for the rendered preflight report.", + ) + + def build_traffic(self) -> BaseTrafficConfig: + """Closed-loop scheduler at the target concurrency, no ramp. + + rampup_seconds is pinned to 0: preflight measures steady-state drift, so + it must be at full concurrency from the start. + """ + return ConcurrentTrafficConfig( + target_concurrent_sessions=self.concurrency, rampup_seconds=0 + ) + + def __post_init__(self) -> None: + if self.concurrency <= 0: + raise ValueError("concurrency must be positive") + if self.num_sessions <= 0: + raise ValueError("num_sessions must be positive") + if not 0.0 <= self.max_unpaired_fraction <= 1.0: + raise ValueError("max_unpaired_fraction must be in [0, 1]") + for name in ( + "delivery_lag_threshold_ms", + "server_pacing_threshold_ms", + "dispatch_drift_threshold_ms", + "input_pacing_threshold_ms", + ): + if getattr(self, name) <= 0: + raise ValueError(f"{name} must be positive") diff --git a/veeksha/core/response.py b/veeksha/core/response.py index 092a9b67..5832f2f0 100644 --- a/veeksha/core/response.py +++ b/veeksha/core/response.py @@ -4,7 +4,7 @@ """ from dataclasses import dataclass, field -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from veeksha.types import ChannelModality @@ -70,9 +70,26 @@ class RequestResult: client_picked_up_at: Optional[float] = ( None # ClientWorker._process_request() on dequeue ) + client_sent_at: Optional[float] = ( + None # t_cs: request handed to the transport (set by wired clients) + ) client_completed_at: Optional[float] = ( None # LLM client send_request() after response ) result_processed_at: Optional[float] = ( None # CompletionWorker._process_result() on entry ) + + # Preflight timing telemetry (the client's own record book). Populated only + # when the client is run with record_preflight_timing=True (the preflight + # harness); left as None on normal benchmark runs so the production path pays + # no cost. The scorer joins these with the server's record book by + # request_id. All monotonic. + chunk_recv_times: Optional[List[float]] = ( + None # t_cr_i: client receipt of each response chunk + ) + # Streaming-input clients only (realtime_tts / vajra / stt): per-input-segment + # send times and the intended pacing deadlines, for input-delivery lag and + # pacing-accuracy metrics. None for single-shot requests. + input_send_times: Optional[List[float]] = None # t_cs_i + input_send_deadlines: Optional[List[float]] = None # intended send instants diff --git a/veeksha/core/tokenizer.py b/veeksha/core/tokenizer.py index f1077a14..5b64141c 100644 --- a/veeksha/core/tokenizer.py +++ b/veeksha/core/tokenizer.py @@ -11,8 +11,6 @@ TypeVar, ) -from transformers import AutoTokenizer - from veeksha.types import ChannelModality RawContent = TypeVar("RawContent") @@ -70,6 +68,11 @@ def build_hf_tokenizer_handle(tokenizer) -> TokenizerHandle[str]: def build_hf_tokenizer_handle_from_model(model: str) -> TokenizerHandle[str]: """Instantiate a Hugging Face tokenizer from a model name and wrap it.""" + # Imported lazily: transformers/tokenizers is heavy and (on free-threaded + # CPython) re-enables the GIL on import, so only pay that cost when an HF + # tokenizer is actually requested. + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True) return build_hf_tokenizer_handle(tokenizer) diff --git a/veeksha/evaluator/performance/base.py b/veeksha/evaluator/performance/base.py index cd7f9a24..d485e890 100644 --- a/veeksha/evaluator/performance/base.py +++ b/veeksha/evaluator/performance/base.py @@ -12,6 +12,7 @@ from veeksha.core.audio_contract import AudioMetricKey from veeksha.core.seeding import SeedManager from veeksha.evaluator.base import BaseEvaluator, EvaluationResult +from veeksha.evaluator.cdf_sketch import CDFSketch from veeksha.logger import init_logger from veeksha.slo.runner import evaluate_and_save_slos from veeksha.types import ChannelModality, ClientType @@ -22,6 +23,18 @@ # the realtime AudioMetricKey contract (audio chunk timestamps + done offsets). _REALTIME_TTS_CLIENT_TYPES = (ClientType.REALTIME_TTS, ClientType.VAJRA_TTS_STREAM) +# Request-lifecycle drift stages (harness scheduling latency), reported in every +# run's summary. Keyed by display name -> the pair of RequestResult timestamps. +_LIFECYCLE_STAGES = { + "Harness Ready-to-Dispatch (ms)": ("scheduler_ready_at", "scheduler_dispatched_at"), + "Harness Dispatch-to-Pickup (ms)": ( + "scheduler_dispatched_at", + "client_picked_up_at", + ), + "Harness Pickup-to-Send (ms)": ("client_picked_up_at", "client_sent_at"), + "Harness Ready-to-Send (ms)": ("scheduler_ready_at", "client_sent_at"), +} + @dataclass class SessionAggregate: @@ -89,6 +102,12 @@ def __init__( self._first_session_start_time: Optional[float] = None self._last_session_start_time: Optional[float] = None + # Request-lifecycle drift sketches (harness scheduling latency). + self._lifecycle_sketches: Dict[str, CDFSketch] = { + name: CDFSketch(name, should_write_to_wandb=False, unit="ms") + for name in _LIFECYCLE_STAGES + } + # streaming support self._stream_trigger = threading.Event() self._stream_stop_event = threading.Event() @@ -221,6 +240,9 @@ def record_request_completed( self.end_time = completed_at self._record_realtime_tts_outcome(response) + # Harness lifecycle drift is about scheduling/dispatch, independent of + # request outcome, so record it before the cancelled/errored returns. + self._accumulate_lifecycle_drift(response) # Update session tracking self._update_session_metrics_for_request( session_id=session_id, @@ -404,9 +426,42 @@ def _get_realtime_tts_summary(self) -> Dict[str, float]: ), } + def _accumulate_lifecycle_drift(self, response: Any) -> None: + """Record the request-lifecycle stage durations (ms) for this request. + + Each stage is skipped if either of its timestamps is missing (e.g. a + request that errored before it was sent has no ``client_sent_at``). + """ + for name, (start_attr, end_attr) in _LIFECYCLE_STAGES.items(): + start = getattr(response, start_attr, None) + end = getattr(response, end_attr, None) + if start is not None and end is not None: + self._lifecycle_sketches[name].put((end - start) * 1000.0) + + def _warn_on_lifecycle_drift(self) -> None: + """Soft-warn if any lifecycle-drift p99 exceeds the configured threshold.""" + threshold = getattr(self.config, "lifecycle_drift_warn_threshold_ms", 0.0) + if threshold is None or threshold <= 0: + return + for name, sketch in self._lifecycle_sketches.items(): + if sketch.sketch.count <= 0: + continue + p99 = sketch.sketch.get_quantile_value(0.99) + if p99 is not None and p99 > threshold: + logger.warning( + "Harness lifecycle drift high: %s p99=%.2fms exceeds %.2fms. " + "The benchmark harness is adding scheduling latency before " + "requests reach the server, which can distort measured " + "server-side timings. Consider more client threads / lower " + "concurrency, or run `veeksha preflight` to certify the box.", + name, + p99, + threshold, + ) + def get_aggregated_summary(self) -> Dict[str, float]: """Get aggregate summary metrics.""" - return { + summary: Dict[str, float] = { "Number of Requests": self.num_requests, "Number of Errored Requests": self.num_errored_requests, "Number of Completed Requests": self.num_completed_requests, @@ -425,6 +480,9 @@ def get_aggregated_summary(self) -> Dict[str, float]: "Observed Session Dispatch Rate": self._session_dispatch_rate(), **self._get_realtime_tts_summary(), } + for sketch in self._lifecycle_sketches.values(): + summary.update(sketch.get_summary()) + return summary def _build_summary_stats(self) -> Dict[str, Any]: """Combine aggregate stats, channel-level metrics, and error code frequencies.""" @@ -443,6 +501,7 @@ def finalize(self) -> EvaluationResult: if self.config.stream_metrics: self._shutdown_metric_streamer() self._finalize_remaining_sessions() + self._warn_on_lifecycle_drift() # Collect metrics from all channel evaluators combined_metrics = self.get_aggregated_summary() diff --git a/veeksha/generator/channel/audio.py b/veeksha/generator/channel/audio.py new file mode 100644 index 00000000..37522506 --- /dev/null +++ b/veeksha/generator/channel/audio.py @@ -0,0 +1,69 @@ +"""Synthetic audio channel generator. + +Produces ``AudioChannelRequestContent`` pointing at a deterministic generated WAV +(a tone or silence). The clip is written once per parameter set to a cache +directory and reused across all requests, so a whole benchmark of STT sessions +decodes the same file once. Content is synthetic -- it exercises the audio-input +path (STT) without a real dataset. +""" + +from __future__ import annotations + +import os +import tempfile +from typing import Any, Optional + +from veeksha.config.generator.channel import AudioChannelGeneratorConfig +from veeksha.core.request_content import AudioChannelRequestContent +from veeksha.core.seeding import SeedManager +from veeksha.generator.channel.base import BaseChannelGenerator + +_CACHE_DIR = os.path.join(tempfile.gettempdir(), "veeksha_synthetic_audio") + + +def _clip_path(config: AudioChannelGeneratorConfig) -> str: + name = ( + f"{config.waveform}_{config.duration_seconds:g}s_{config.sample_rate}hz" + f"_{config.frequency_hz:g}.wav" + ) + return os.path.join(_CACHE_DIR, name) + + +def _ensure_clip(config: AudioChannelGeneratorConfig) -> str: + """Write the synthetic WAV once (idempotent) and return its path.""" + path = _clip_path(config) + if os.path.exists(path): + return path + + import numpy as np + import soundfile as sf + + n = int(config.duration_seconds * config.sample_rate) + if config.waveform == "silence": + samples = np.zeros(n, dtype="float32") + else: # sine + t = np.arange(n) / config.sample_rate + samples = (0.1 * np.sin(2.0 * np.pi * config.frequency_hz * t)).astype( + "float32" + ) + + os.makedirs(_CACHE_DIR, exist_ok=True) + tmp = f"{path}.{os.getpid()}.tmp" + # format is explicit -- the .tmp name has no extension for soundfile to infer. + sf.write(tmp, samples, config.sample_rate, subtype="PCM_16", format="WAV") + os.replace(tmp, path) # atomic; safe under the thundering herd at run start + return path + + +class AudioChannelGenerator(BaseChannelGenerator): + def __init__( + self, + config: AudioChannelGeneratorConfig, + seed_manager: SeedManager, + tokenizer_handle: Optional[Any] = None, + ) -> None: + super().__init__(config, seed_manager) + self._config = config + + def generate_content(self, is_root: bool = False) -> AudioChannelRequestContent: + return AudioChannelRequestContent(input_audio=_ensure_clip(self._config)) diff --git a/veeksha/generator/channel/registry.py b/veeksha/generator/channel/registry.py index 016980dd..32dde76a 100644 --- a/veeksha/generator/channel/registry.py +++ b/veeksha/generator/channel/registry.py @@ -1,3 +1,4 @@ +from veeksha.generator.channel.audio import AudioChannelGenerator from veeksha.generator.channel.text import TextChannelGenerator from veeksha.types import ChannelModality from veeksha.types.base_registry import BaseRegistry @@ -10,3 +11,4 @@ def get_key_from_str(cls, key_str: str) -> ChannelModality: ChannelGeneratorRegistry.register(ChannelModality.TEXT, TextChannelGenerator) +ChannelGeneratorRegistry.register(ChannelModality.AUDIO, AudioChannelGenerator) diff --git a/veeksha/preflight/__init__.py b/veeksha/preflight/__init__.py new file mode 100644 index 00000000..5a6df7eb --- /dev/null +++ b/veeksha/preflight/__init__.py @@ -0,0 +1,42 @@ +"""Preflight timing-fidelity validation. + +Before trusting a benchmark's numbers we certify that the *harness itself* keeps +time honestly: that requests are dispatched when the schedule says, that they +reach the server when the client thinks they were sent, and that streamed +responses arrive when the server says it sent them. We do this by pointing the +real scheduler -> dispatch -> client path at deterministic mock servers (run in +separate processes to keep their emit schedules out of the clients' way) that +stamp ground-truth send/receive times, then scoring the drift between what each +side observed. + +Notation used throughout (all ``time.monotonic()``): + +* ``t_cs`` -- client-sent: request handed to the transport +* ``t_sr`` -- server-received: mock server accepted the request +* ``t_ss_i`` -- server-sent: mock emitted response chunk ``i`` +* ``t_cr_i`` -- client-received: client observed response chunk ``i`` + +Drifts we track as metrics (p50/p99, gated on thresholds): + +* request-delivery lag ``t_sr - t_cs`` +* response-delivery lag ``t_cr_i - t_ss_i`` +* server pacing fidelity ``t_ss_1 - t_sr`` vs ttfc, ``t_ss_{i+1} - t_ss_i`` vs tpoc +* dispatch drift ``scheduler_dispatched_at - scheduler_ready_at`` and + the end-to-end ``t_cs - scheduler_ready_at`` +""" + +from veeksha.preflight.models import ( + MetricSummary, + ScoreReport, + ServerRequestRecord, +) +from veeksha.preflight.scorer import percentile, score, summarize + +__all__ = [ + "MetricSummary", + "ScoreReport", + "ServerRequestRecord", + "percentile", + "score", + "summarize", +] diff --git a/veeksha/preflight/drivers.py b/veeksha/preflight/drivers.py new file mode 100644 index 00000000..0587e072 --- /dev/null +++ b/veeksha/preflight/drivers.py @@ -0,0 +1,485 @@ +"""Preflight drivers: run the REAL benchmark loop against a mock server. + +The whole point of preflight is to measure the *actual* harness, so a driver +reuses veeksha's real components -- session generator, traffic scheduler, +dispatch/client/completion workers via ``benchmark._run_main_loop`` -- and only +substitutes (a) a deterministic mock server for the LLM endpoint and (b) a +capturing evaluator that keeps every ``RequestResult`` so the scorer can read +the paired client/server timestamps. + +A word-split tokenizer is injected rather than the client's default HuggingFace +one, so preflight needs no ``transformers``/``tokenizers`` install and runs on a +bare free-threaded interpreter. + +Each ``run_*_preflight`` takes its category workload config (input shape + mock +timing), the shared traffic scheduler, and the session count; it forces the +endpoint to the spawned mock and turns on ``record_preflight_timing``. +""" + +from __future__ import annotations + +from typing import List + +from veeksha.benchmark import _run_main_loop +from veeksha.benchmark_utils import build_evaluator +from veeksha.client import ClientRegistry +from veeksha.config.benchmark import BenchmarkConfig +from veeksha.config.client import ( + BaseClientConfig, + OpenAIChatCompletionsClientConfig, + OpenAICompletionsClientConfig, + RealtimeTTSClientConfig, + STTClientConfig, + TextPacingConfig, + TTSClientConfig, + VajraTTSStreamClientConfig, +) +from veeksha.config.evaluator import PerformanceEvaluatorConfig +from veeksha.config.generator.channel import ( + AudioChannelGeneratorConfig, + TextChannelGeneratorConfig, +) +from veeksha.config.generator.length import FixedLengthGeneratorConfig +from veeksha.config.generator.session import SyntheticSessionGeneratorConfig +from veeksha.config.generator.session_graph import ( + SingleRequestSessionGraphGeneratorConfig, +) +from veeksha.config.preflight import ( + PreflightSttCheckConfig, + PreflightTextCheckConfig, + PreflightTtsCheckConfig, +) +from veeksha.config.runtime import RuntimeConfig +from veeksha.config.traffic import BaseTrafficConfig +from veeksha.core.seeding import SeedManager +from veeksha.core.tokenizer import TokenizerHandle, TokenizerProvider +from veeksha.generator.session.registry import SessionGeneratorRegistry +from veeksha.logger import init_logger +from veeksha.preflight import scorer +from veeksha.preflight.models import ScoreReport +from veeksha.preflight.spawn import ( + MockServerHandle, + spawn_mock_chat_server, + spawn_mock_completions_server, + spawn_mock_realtime_tts_server, + spawn_mock_stt_server, + spawn_mock_tts_server, + spawn_mock_vajra_tts_server, +) +from veeksha.traffic.registry import TrafficSchedulerRegistry +from veeksha.types import ChannelModality + +logger = init_logger(__name__) + +# HTTP-TTS: the mock emits chunks of this size and the client reads with a +# matching chunk_size, so each server chunk is read 1:1 for the join. +_TTS_CHUNK_BYTES = 1024 + + +def _preflight_encode(text: str): + """Whitespace tokenizer that round-trips integer words stably. + + The prompt generator checks ``encode(decode(x)) == x`` on integer seeds, so a + positional word-split encoder (which maps to range(n)) never stabilises. Here + an integer word maps to itself and non-integer words (e.g. the mock's response + text, which the client also encodes) map to a deterministic id. + """ + ids = [] + for word in text.split(): + if word.lstrip("-").isdigit(): + ids.append(int(word)) + else: + ids.append(sum(map(ord, word))) + return ids + + +def _build_preflight_tokenizer_provider(model: str) -> TokenizerProvider: + text_handle = TokenizerHandle( + # Each token decodes to " " -- the trailing space is a per-token + # boundary so PromptStringGenerator (which tiles decoded tokens with + # "".join) yields whitespace-separated words. Without it a prompt is one + # space-less blob that segment_text treats as a single streamed delta. + count_tokens=lambda text: len(text.split()), + decode=lambda ids: "".join(f"{int(i)} " for i in ids), + encode=_preflight_encode, + get_vocab=lambda: list(range(10_000)), + ) + # The audio channel generator ignores its tokenizer handle; a passthrough + # satisfies the synthetic generator's per-channel lookup for AUDIO input. + audio_handle = TokenizerHandle( + count_tokens=lambda _: 0, + decode=lambda ids: "", + encode=lambda _: [], + ) + return TokenizerProvider( + {ChannelModality.TEXT: text_handle, ChannelModality.AUDIO: audio_handle}, + model_name=model, + ) + + +def _text_session_config(input_tokens: int) -> SyntheticSessionGeneratorConfig: + return SyntheticSessionGeneratorConfig( + session_graph=SingleRequestSessionGraphGeneratorConfig(), + channels=[ + TextChannelGeneratorConfig( + body_length_generator=FixedLengthGeneratorConfig(value=input_tokens) + ) + ], + ) + + +def _audio_session_config( + duration_seconds: float, sample_rate: int +) -> SyntheticSessionGeneratorConfig: + return SyntheticSessionGeneratorConfig( + session_graph=SingleRequestSessionGraphGeneratorConfig(), + channels=[ + AudioChannelGeneratorConfig( + duration_seconds=duration_seconds, sample_rate=sample_rate + ) + ], + ) + + +def _stream_pacing(tokens_per_second: float, tokens_per_delta: int) -> TextPacingConfig: + return TextPacingConfig( + tokens_per_second=tokens_per_second, + tokens_per_delta=tokens_per_delta, + initial_delay_s=0.0, + ) + + +class CapturingEvaluator: + """Wraps a real evaluator, keeping every completed ``RequestResult``. + + All calls the workers / monitor loop make delegate to the wrapped evaluator so + termination behaviour is identical; we only tap ``record_request_completed`` + to stash the full result for scoring. + """ + + def __init__(self, wrapped) -> None: + self._wrapped = wrapped + self.results: List[object] = [] + + def record_request_completed( + self, request_id, session_id, completed_at, response, error=None + ) -> None: + self.results.append(response) + return self._wrapped.record_request_completed( + request_id, session_id, completed_at, response, error + ) + + def __getattr__(self, name): + return getattr(self._wrapped, name) + + +def _build_benchmark_config( + client_config: BaseClientConfig, + session_config: SyntheticSessionGeneratorConfig, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> BenchmarkConfig: + return BenchmarkConfig( + output_dir=output_dir, + session_generator=session_config, + traffic_scheduler=traffic_scheduler, + client=client_config, + runtime=RuntimeConfig(max_sessions=num_sessions), + server=None, + # Preflight scores its own drift metrics; suppress the evaluator's dump. + evaluators=[PerformanceEvaluatorConfig(stream_metrics=False)], + ) + + +def _run_capturing(benchmark_config: BenchmarkConfig) -> List[object]: + """Run the real main loop for one BenchmarkConfig; return captured results. + + Mirrors ``benchmark._run_benchmark`` construction, but forces a word-split + tokenizer and wraps the evaluator to capture results. + """ + import time + + seed_manager = SeedManager(benchmark_config.seed) + tokenizer_provider = _build_preflight_tokenizer_provider( + benchmark_config.client.model + ) + session_generator = SessionGeneratorRegistry.get( + benchmark_config.session_generator.get_type(), + config=benchmark_config.session_generator, + seed_manager=seed_manager, + tokenizer_provider=tokenizer_provider, + append_min_tokens_instruction=False, + ) + traffic_scheduler = TrafficSchedulerRegistry.get( + benchmark_config.traffic_scheduler.get_type(), + config=benchmark_config.traffic_scheduler, + seed_manager=seed_manager, + ) + client = ClientRegistry.get( + benchmark_config.client.get_type(), + config=benchmark_config.client, + tokenizer_provider=tokenizer_provider, + ) + + benchmark_start_time = time.monotonic() + traffic_scheduler.reset_reference_time() + + evaluator = build_evaluator( + benchmark_config, + seed_manager=seed_manager, + session_generator=session_generator, + benchmark_start_time=benchmark_start_time, + ) + capturing = CapturingEvaluator(evaluator) + + _run_main_loop( + session_generator=session_generator, + traffic_scheduler=traffic_scheduler, + evaluator=capturing, + client=client, + runtime_config=benchmark_config.runtime, + benchmark_start_time=benchmark_start_time, + ) + return capturing.results + + +def _score_run( + client_config: BaseClientConfig, + session_config: SyntheticSessionGeneratorConfig, + server: MockServerHandle, + *, + ttfc_ms: float, + tpoc_ms: float, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> ScoreReport: + """Drive one client config against a running mock and score the drift.""" + benchmark_config = _build_benchmark_config( + client_config, session_config, traffic_scheduler, num_sessions, output_dir + ) + results = _run_capturing(benchmark_config) + server_records = server.fetch_records() + return scorer.score(results, server_records, ttfc_ms=ttfc_ms, tpoc_ms=tpoc_ms) + + +def _log(kind: str, num_sessions: int, server: MockServerHandle) -> None: + logger.info( + "Preflight %s: %d sessions vs mock %s", kind, num_sessions, server.api_base + ) + + +# --------------------------------------------------------------------------- +# text group: chat + completions +# --------------------------------------------------------------------------- + + +def run_text_preflight( + cfg: PreflightTextCheckConfig, + *, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> ScoreReport: + """Chat (streaming SSE) path.""" + with spawn_mock_chat_server( + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + num_chunks=cfg.num_response_chunks, + ) as server: + _log("text", num_sessions, server) + client_config = OpenAIChatCompletionsClientConfig( + api_base=server.api_base, + api_key="preflight", + model="preflight-mock", + record_preflight_timing=True, + ) + return _score_run( + client_config, + _text_session_config(cfg.input_tokens), + server, + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + traffic_scheduler=traffic_scheduler, + num_sessions=num_sessions, + output_dir=output_dir, + ) + + +def run_completions_preflight( + cfg: PreflightTextCheckConfig, + *, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> ScoreReport: + """Completions (non-streaming) path -- one response, so no tpoc metric.""" + with spawn_mock_completions_server(ttfc_ms=cfg.server_ttfc_ms) as server: + _log("completions", num_sessions, server) + client_config = OpenAICompletionsClientConfig( + api_base=server.api_base, + api_key="preflight", + model="preflight-mock", + record_preflight_timing=True, + ) + return _score_run( + client_config, + _text_session_config(cfg.input_tokens), + server, + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=0.0, + traffic_scheduler=traffic_scheduler, + num_sessions=num_sessions, + output_dir=output_dir, + ) + + +# --------------------------------------------------------------------------- +# tts group: tts (HTTP) + realtime_tts + vajra (WS) +# --------------------------------------------------------------------------- + + +def run_tts_preflight( + cfg: PreflightTtsCheckConfig, + *, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> ScoreReport: + """TTS (HTTP streaming raw audio) path.""" + with spawn_mock_tts_server( + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + num_chunks=cfg.num_response_chunks, + chunk_bytes=_TTS_CHUNK_BYTES, + ) as server: + _log("tts", num_sessions, server) + client_config = TTSClientConfig( + api_base=server.api_base, + api_key="preflight", + model="preflight-mock", + voice_id="preflight", + raw_pcm=True, + chunk_size=_TTS_CHUNK_BYTES, + record_preflight_timing=True, + ) + return _score_run( + client_config, + _text_session_config(cfg.input_tokens), + server, + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + traffic_scheduler=traffic_scheduler, + num_sessions=num_sessions, + output_dir=output_dir, + ) + + +def run_realtime_tts_preflight( + cfg: PreflightTtsCheckConfig, + *, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> ScoreReport: + """Realtime-TTS (WebSocket) path.""" + with spawn_mock_realtime_tts_server( + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + num_chunks=cfg.num_response_chunks, + ) as server: + _log("realtime_tts", num_sessions, server) + client_config = RealtimeTTSClientConfig( + api_base=server.api_base, + api_key="preflight", + model="preflight-mock", + voice_id="preflight", + pacing=_stream_pacing(cfg.input_pacing_tps, cfg.input_chunk_tokens), + record_preflight_timing=True, + ) + return _score_run( + client_config, + _text_session_config(cfg.input_tokens), + server, + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + traffic_scheduler=traffic_scheduler, + num_sessions=num_sessions, + output_dir=output_dir, + ) + + +def run_vajra_tts_preflight( + cfg: PreflightTtsCheckConfig, + *, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> ScoreReport: + """Vajra TTS-stream (WebSocket binary PCM) path.""" + with spawn_mock_vajra_tts_server( + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + num_chunks=cfg.num_response_chunks, + ) as server: + _log("vajra_tts_stream", num_sessions, server) + client_config = VajraTTSStreamClientConfig( + api_base=server.api_base, + api_key="preflight", + model="preflight-mock", + voice_id="preflight", + pacing=_stream_pacing(cfg.input_pacing_tps, cfg.input_chunk_tokens), + record_preflight_timing=True, + ) + return _score_run( + client_config, + _text_session_config(cfg.input_tokens), + server, + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + traffic_scheduler=traffic_scheduler, + num_sessions=num_sessions, + output_dir=output_dir, + ) + + +# --------------------------------------------------------------------------- +# stt group +# --------------------------------------------------------------------------- + + +def run_stt_preflight( + cfg: PreflightSttCheckConfig, + *, + traffic_scheduler: BaseTrafficConfig, + num_sessions: int, + output_dir: str, +) -> ScoreReport: + """STT (WebSocket audio-in) path -- synthetic audio input, transcript out.""" + with spawn_mock_stt_server( + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + num_chunks=cfg.num_response_chunks, + ) as server: + _log("stt", num_sessions, server) + client_config = STTClientConfig( + api_base=server.api_base, + api_key="preflight", + model="preflight-mock", + provider="vllm_realtime", + sample_rate=cfg.sample_rate, + ws_chunk_size=cfg.input_chunk_bytes, + ws_realtime_pacing=True, + record_preflight_timing=True, + ) + return _score_run( + client_config, + _audio_session_config(cfg.input_seconds, cfg.sample_rate), + server, + ttfc_ms=cfg.server_ttfc_ms, + tpoc_ms=cfg.server_tpoc_ms, + traffic_scheduler=traffic_scheduler, + num_sessions=num_sessions, + output_dir=output_dir, + ) diff --git a/veeksha/preflight/models.py b/veeksha/preflight/models.py new file mode 100644 index 00000000..75a76300 --- /dev/null +++ b/veeksha/preflight/models.py @@ -0,0 +1,97 @@ +"""Plain data structures shared across the preflight package. + +Kept dependency-free (no I/O, no veeksha.core imports beyond typing) so the +scorer and its tests can construct them directly. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + + +@dataclass +class ServerRequestRecord: + """Ground-truth timestamps a mock server recorded for one request. + + All timestamps are ``time.monotonic()`` from the *server* process. Because + ``time.monotonic()`` maps to a machine-wide clock (CLOCK_MONOTONIC / mach + absolute time), these are directly comparable to the client's stamps as long + as both run on the same host -- which the preflight harness guarantees. + """ + + request_id: int + server_recv_time: float # t_sr (connection / request accept) + server_send_times: List[float] = field(default_factory=list) # t_ss_i + # t_sr_i: per-input-segment receipt times. Only populated by streaming-input + # mocks (realtime_tts / vajra / stt); empty for single-shot HTTP requests. + input_recv_times: List[float] = field(default_factory=list) + # When the server began emitting the response. The ttfc pacing metric is + # anchored here, so it isn't polluted by a streaming-input phase. HTTP mocks + # leave this None (the scorer falls back to server_recv_time, since they + # respond immediately); WS mocks set it after the input completes. + response_start_time: Optional[float] = None + + def to_json(self) -> dict: + return { + "request_id": self.request_id, + "server_recv_time": self.server_recv_time, + "server_send_times": self.server_send_times, + "input_recv_times": self.input_recv_times, + "response_start_time": self.response_start_time, + } + + @classmethod + def from_json(cls, d: dict) -> "ServerRequestRecord": + response_start_time = d.get("response_start_time") + return cls( + request_id=int(d["request_id"]), + server_recv_time=float(d["server_recv_time"]), + server_send_times=[float(x) for x in d.get("server_send_times", [])], + input_recv_times=[float(x) for x in d.get("input_recv_times", [])], + response_start_time=( + None if response_start_time is None else float(response_start_time) + ), + ) + + +@dataclass +class MetricSummary: + """Distribution summary for one drift metric (all values in ms).""" + + name: str + count: int + p50: float + p99: float + mean: float + minimum: float + maximum: float + + def to_json(self) -> dict: + return { + "name": self.name, + "count": self.count, + "p50": self.p50, + "p99": self.p99, + "mean": self.mean, + "min": self.minimum, + "max": self.maximum, + } + + +@dataclass +class ScoreReport: + """All drift metrics for one preflight run, plus bookkeeping.""" + + metrics: Dict[str, MetricSummary] = field(default_factory=dict) + n_requests: int = 0 + n_paired_requests: int = 0 # had both client + server records + unpaired_fraction: float = 0.0 + + def to_json(self) -> dict: + return { + "n_requests": self.n_requests, + "n_paired_requests": self.n_paired_requests, + "unpaired_fraction": self.unpaired_fraction, + "metrics": {k: v.to_json() for k, v in self.metrics.items()}, + } diff --git a/veeksha/preflight/report.py b/veeksha/preflight/report.py new file mode 100644 index 00000000..7d3d475b --- /dev/null +++ b/veeksha/preflight/report.py @@ -0,0 +1,92 @@ +"""Render a preflight run (scores + verdict) as a plain-text report.""" + +from __future__ import annotations + +import math +import os +from typing import List + +from veeksha.preflight.models import ScoreReport +from veeksha.preflight.validator import ValidationResult + +_REPORT_FILENAME = "preflight_report.txt" + + +def _fmt(value: float) -> str: + if value is None or (isinstance(value, float) and math.isnan(value)): + return "n/a" + return f"{value:.3f}" + + +def _table(rows: List[List[str]], headers: List[str]) -> str: + cols = list(zip(*([headers] + rows))) if rows else [[h] for h in headers] + widths = [max(len(str(c)) for c in col) for col in cols] + line = " ".join(h.ljust(widths[i]) for i, h in enumerate(headers)) + sep = " ".join("-" * widths[i] for i in range(len(headers))) + out = [line, sep] + for row in rows: + out.append(" ".join(str(c).ljust(widths[i]) for i, c in enumerate(row))) + return "\n".join(out) + + +def render_report( + score_report: ScoreReport, + validation: ValidationResult, + *, + title: str = "Preflight measurement-fidelity check", +) -> str: + lines: List[str] = [] + lines.append("=" * 72) + lines.append(title) + lines.append("=" * 72) + lines.append("") + lines.append(f"VERDICT: {validation.verdict.upper()}") + lines.append( + f"requests: {score_report.n_requests} " + f"paired: {score_report.n_paired_requests} " + f"unpaired: {score_report.unpaired_fraction:.3%}" + ) + lines.append("") + + # --- gates --- + lines.append("Gates (p99 vs threshold):") + gate_rows = [ + [ + g.name, + g.category, + _fmt(g.p99), + _fmt(g.threshold), + "PASS" if g.passed else "FAIL", + ] + for g in validation.gates + ] + lines.append(_table(gate_rows, ["gate", "blames", "p99", "threshold", "result"])) + lines.append("") + + # --- full metric distributions --- + lines.append("Metrics (ms):") + metric_rows = [] + for name in sorted(score_report.metrics): + m = score_report.metrics[name] + metric_rows.append( + [ + name, + str(m.count), + _fmt(m.p50), + _fmt(m.p99), + _fmt(m.mean), + _fmt(m.maximum), + ] + ) + lines.append(_table(metric_rows, ["metric", "count", "p50", "p99", "mean", "max"])) + lines.append("") + return "\n".join(lines) + + +def write_report(text: str, output_dir: str) -> str: + """Write the rendered report to ``output_dir`` and return the path.""" + os.makedirs(output_dir, exist_ok=True) + path = os.path.join(output_dir, _REPORT_FILENAME) + with open(path, "w") as f: + f.write(text) + return path diff --git a/veeksha/preflight/runner.py b/veeksha/preflight/runner.py new file mode 100644 index 00000000..b1901f9d --- /dev/null +++ b/veeksha/preflight/runner.py @@ -0,0 +1,113 @@ +"""CLI entry point for ``veeksha preflight``. + +Runs each configured check, scores the timing drift, gates it into a verdict, +prints and writes a per-check report (under ``//``), and +exits non-zero unless every check comes back honest. +""" + +from __future__ import annotations + +import os +import sys +from typing import List + +from veeksha.config.preflight import PreflightCheckConfig +from veeksha.logger import init_logger +from veeksha.preflight import validator +from veeksha.preflight.drivers import ( + run_completions_preflight, + run_realtime_tts_preflight, + run_stt_preflight, + run_text_preflight, + run_tts_preflight, + run_vajra_tts_preflight, +) +from veeksha.preflight.models import ScoreReport +from veeksha.preflight.report import render_report, write_report + +logger = init_logger(__name__) + + +def _gate_and_render( + config: PreflightCheckConfig, report: ScoreReport, title: str, output_dir: str +) -> bool: + """Validate one check's report, print + write it, return True iff PASS.""" + result = validator.run_validation( + report, + delivery_lag_threshold_ms=config.delivery_lag_threshold_ms, + server_pacing_threshold_ms=config.server_pacing_threshold_ms, + dispatch_drift_threshold_ms=config.dispatch_drift_threshold_ms, + input_pacing_threshold_ms=config.input_pacing_threshold_ms, + max_unpaired_fraction=config.max_unpaired_fraction, + ) + text = render_report(report, result, title=title) + path = write_report(text, output_dir) + print(text) + print(f"Report written to {path}") + return result.is_pass + + +def _run_one_check(config: PreflightCheckConfig) -> bool: + """Run every enabled check for one config; return True iff all passed. + + Each check gets its own ``//`` subdirectory (report + + metrics) so the per-modality outputs don't overwrite one another. + """ + all_passed = True + common = dict( + traffic_scheduler=config.build_traffic(), + num_sessions=config.num_sessions, + ) + + # (enabled, runner, group config, dir slug, display name) + checks = [ + (config.check_text, run_text_preflight, config.text, "chat", "text (chat)"), + ( + config.check_completions, + run_completions_preflight, + config.text, + "completions", + "completions", + ), + (config.check_tts, run_tts_preflight, config.tts, "tts", "tts"), + ( + config.check_realtime_tts, + run_realtime_tts_preflight, + config.tts, + "realtime_tts", + "realtime_tts", + ), + ( + config.check_vajra_tts, + run_vajra_tts_preflight, + config.tts, + "vajra_tts_stream", + "vajra_tts_stream", + ), + (config.check_stt, run_stt_preflight, config.stt, "stt", "stt"), + ] + for enabled, runner, group_cfg, slug, name in checks: + if not enabled: + continue + logger.info("Running preflight %s check", name) + check_output_dir = os.path.join(config.output_dir, slug) + report = runner(group_cfg, output_dir=check_output_dir, **common) + all_passed &= _gate_and_render( + config, + report, + f"Preflight measurement fidelity: {name}", + check_output_dir, + ) + + return all_passed + + +def run_preflight_cli(configs: List[PreflightCheckConfig]) -> None: + overall_passed = True + for config in configs: + overall_passed = _run_one_check(config) and overall_passed + + if not overall_passed: + # Non-zero exit so CI/scripts can gate a benchmark on a clean preflight. + # FAIL and SERVER_AT_CAPACITY both count as "did not pass". + sys.exit(1) diff --git a/veeksha/preflight/scorer.py b/veeksha/preflight/scorer.py new file mode 100644 index 00000000..f98d0a25 --- /dev/null +++ b/veeksha/preflight/scorer.py @@ -0,0 +1,216 @@ +"""Pure drift math over paired client/server timestamps. + +No I/O and no framework imports: everything here operates on plain numbers and +the light dataclasses in :mod:`veeksha.preflight.models`, so it is trivially +unit-testable. ``score`` consumes ``RequestResult``-shaped objects (duck-typed +via attribute access) plus the server's ground-truth records. +""" + +from __future__ import annotations + +import math +from typing import Dict, Iterable, List, Mapping, Optional + +from veeksha.preflight.models import MetricSummary, ScoreReport, ServerRequestRecord + +# Metric names -- referenced by the validator's gates so they stay in sync. +M_REQUEST_DELIVERY = "request_delivery_ms" # t_sr - t_cs +M_RESPONSE_DELIVERY = "response_delivery_ms" # t_cr_i - t_ss_i +M_SERVER_TTFC_ABS_ERR = "server_ttfc_abs_error_ms" # |(t_ss_0 - t_sr) - ttfc| +M_SERVER_TPOC_ABS_ERR = "server_tpoc_abs_error_ms" # |gap - tpoc| +M_CLIENT_TTFC = "client_observed_ttfc_ms" # t_cr_0 - t_cs (informational) +M_CLIENT_TPOC = "client_observed_tpoc_ms" # t_cr_{i+1} - t_cr_i (informational) +M_LIFECYCLE_READY_TO_DISPATCH = "lifecycle_ready_to_dispatch_ms" +M_LIFECYCLE_DISPATCH_TO_PICKUP = "lifecycle_dispatch_to_pickup_ms" +M_LIFECYCLE_PICKUP_TO_SEND = "lifecycle_pickup_to_send_ms" +M_LIFECYCLE_READY_TO_SEND = "lifecycle_ready_to_send_ms" # end-to-end (t_cs - ready) +# Streaming-input clients only (realtime_tts / vajra / stt): +M_INPUT_DELIVERY = "input_delivery_ms" # t_sr_i - t_cs_i (per input segment) +M_INPUT_PACING_ABS_ERR = "input_pacing_abs_error_ms" # |t_cs_i - deadline_i| + +_MS = 1000.0 + + +def percentile(values: List[float], q: float) -> float: + """Linear-interpolation percentile (like numpy's default). NaN if empty.""" + xs = sorted(v for v in values if not math.isnan(v)) + if not xs: + return float("nan") + if len(xs) == 1: + return xs[0] + rank = (q / 100.0) * (len(xs) - 1) + lo = math.floor(rank) + hi = math.ceil(rank) + if lo == hi: + return xs[int(lo)] + frac = rank - lo + return xs[int(lo)] * (1.0 - frac) + xs[int(hi)] * frac + + +def summarize(name: str, values: Iterable[float]) -> MetricSummary: + xs = [v for v in values if not math.isnan(v)] + if not xs: + nan = float("nan") + return MetricSummary(name, 0, nan, nan, nan, nan, nan) + return MetricSummary( + name=name, + count=len(xs), + p50=percentile(xs, 50), + p99=percentile(xs, 99), + mean=sum(xs) / len(xs), + minimum=min(xs), + maximum=max(xs), + ) + + +def _append(bucket: Dict[str, List[float]], name: str, value: float) -> None: + bucket.setdefault(name, []).append(value) + + +def score( + results: Iterable[object], + server_records: Mapping[int, ServerRequestRecord], + ttfc_ms: float, + tpoc_ms: float, +) -> ScoreReport: + """Compute every drift metric for one preflight run. + + Args: + results: RequestResult-shaped objects (need ``request_id`` plus the + lifecycle/preflight timestamp attributes; missing ones are skipped). + server_records: request_id -> ServerRequestRecord ground truth. + ttfc_ms: configured server first-chunk delay, for pacing-fidelity error. + tpoc_ms: configured server inter-chunk delay, for pacing-fidelity error. + """ + buckets: Dict[str, List[float]] = {} + n_requests = 0 + n_paired = 0 + + for r in results: + n_requests += 1 + request_id = getattr(r, "request_id", None) + client_send_time = getattr(r, "client_sent_at", None) # t_cs + ready_time = getattr(r, "scheduler_ready_at", None) + dispatched_time = getattr(r, "scheduler_dispatched_at", None) + pickup_time = getattr(r, "client_picked_up_at", None) + client_recv_times: Optional[List[float]] = getattr( + r, "chunk_recv_times", None + ) # t_cr_i + input_send_times: Optional[List[float]] = getattr( + r, "input_send_times", None + ) # t_cs_i + input_send_deadlines: Optional[List[float]] = getattr( + r, "input_send_deadlines", None + ) + + server_record = ( + server_records.get(request_id) if request_id is not None else None + ) + + # --- dispatch drift (harness-only; no server record needed) --- + if ready_time is not None and dispatched_time is not None: + _append( + buckets, + M_LIFECYCLE_READY_TO_DISPATCH, + (dispatched_time - ready_time) * _MS, + ) + if dispatched_time is not None and pickup_time is not None: + _append( + buckets, + M_LIFECYCLE_DISPATCH_TO_PICKUP, + (pickup_time - dispatched_time) * _MS, + ) + if pickup_time is not None and client_send_time is not None: + _append( + buckets, + M_LIFECYCLE_PICKUP_TO_SEND, + (client_send_time - pickup_time) * _MS, + ) + if ready_time is not None and client_send_time is not None: + _append( + buckets, + M_LIFECYCLE_READY_TO_SEND, + (client_send_time - ready_time) * _MS, + ) + + # --- response delivery: client receipt (t_cr_i) vs server send (t_ss_i) --- + # Joined from the two record books by index within this request (relies + # on 1:1 chunk ordering, which the paced localhost mocks hold). + if ( + client_recv_times + and server_record is not None + and server_record.server_send_times + ): + for client_recv_time, server_send_time in zip( + client_recv_times, server_record.server_send_times + ): + _append( + buckets, + M_RESPONSE_DELIVERY, + (client_recv_time - server_send_time) * _MS, + ) + + # --- streaming-input pacing accuracy (client-only): t_cs_i vs deadline --- + if input_send_times and input_send_deadlines: + for send_time, deadline in zip(input_send_times, input_send_deadlines): + _append( + buckets, M_INPUT_PACING_ABS_ERR, abs(send_time - deadline) * _MS + ) + + # --- client-observed ttfc / tpoc (informational) --- + if client_recv_times and client_send_time is not None: + _append( + buckets, M_CLIENT_TTFC, (client_recv_times[0] - client_send_time) * _MS + ) + if client_recv_times and len(client_recv_times) >= 2: + for i in range(1, len(client_recv_times)): + _append( + buckets, + M_CLIENT_TPOC, + (client_recv_times[i] - client_recv_times[i - 1]) * _MS, + ) + + # --- server-side ground truth: request delivery + pacing fidelity --- + if server_record is None: + continue + n_paired += 1 + + server_recv_time = server_record.server_recv_time # t_sr + if client_send_time is not None: + _append( + buckets, + M_REQUEST_DELIVERY, + (server_recv_time - client_send_time) * _MS, + ) + + # streaming-input delivery: server receipt (t_sr_i) vs client send (t_cs_i) + if input_send_times and server_record.input_recv_times: + for send_time, server_input_recv in zip( + input_send_times, server_record.input_recv_times + ): + _append( + buckets, M_INPUT_DELIVERY, (server_input_recv - send_time) * _MS + ) + + server_send_times = server_record.server_send_times # t_ss_i + if server_send_times: + # Anchor ttfc at when the server began responding (== t_sr for + # single-shot HTTP; after the input phase for streaming-input WS). + response_start = server_record.response_start_time + if response_start is None: + response_start = server_recv_time + ttfc_actual = (server_send_times[0] - response_start) * _MS + _append(buckets, M_SERVER_TTFC_ABS_ERR, abs(ttfc_actual - ttfc_ms)) + for i in range(1, len(server_send_times)): + gap = (server_send_times[i] - server_send_times[i - 1]) * _MS + _append(buckets, M_SERVER_TPOC_ABS_ERR, abs(gap - tpoc_ms)) + + report = ScoreReport( + metrics={name: summarize(name, vals) for name, vals in buckets.items()}, + n_requests=n_requests, + n_paired_requests=n_paired, + unpaired_fraction=( + 0.0 if n_requests == 0 else (n_requests - n_paired) / n_requests + ), + ) + return report diff --git a/veeksha/preflight/servers/__init__.py b/veeksha/preflight/servers/__init__.py new file mode 100644 index 00000000..74f15451 --- /dev/null +++ b/veeksha/preflight/servers/__init__.py @@ -0,0 +1,7 @@ +"""Deterministic mock servers for preflight timing validation. + +Each server runs as its own process (see :mod:`veeksha.preflight.spawn`) so its +emit schedule stays punctual and out of the way of the veeksha clients under +test. Servers stamp ground-truth send/receive times and expose them at +``GET /preflight/records``. (Dedicated-core pinning may be added later.) +""" diff --git a/veeksha/preflight/servers/base_mock.py b/veeksha/preflight/servers/base_mock.py new file mode 100644 index 00000000..a3f0c20c --- /dev/null +++ b/veeksha/preflight/servers/base_mock.py @@ -0,0 +1,89 @@ +"""Shared skeleton for preflight mock servers. + +Handles the parts every mock has in common -- ``/health``, ``/preflight/records``, +request-id / client-sent-time header parsing, the ground-truth record book, and +the accept loop -- leaving each concrete mock to implement only ``handle_post``: +how it paces and shapes its response. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from typing import Dict, Optional, Tuple + +from veeksha.preflight.models import ServerRequestRecord +from veeksha.preflight.servers.base_server import ( + HttpRequest, + close_writer, + read_http_request, + write_response, +) + + +class BaseMockServer: + def __init__(self, host: str, port: int) -> None: + self.host = host + self.port = port + self.records: Dict[int, ServerRequestRecord] = {} + self._synthetic = -1 + + def _next_synthetic_id(self) -> int: + rid = self._synthetic + self._synthetic -= 1 + return rid + + def _request_id(self, header_value: Optional[str]) -> int: + if header_value is None: + return self._next_synthetic_id() + try: + return int(header_value) + except ValueError: + return self._next_synthetic_id() + + def open_record(self, req: HttpRequest) -> Tuple[int, ServerRequestRecord]: + """Stamp receipt (t_sr), key by request id, and start a record. + + Call this first thing in ``handle_post``, before any response work. The + request id (``X-Veeksha-Request-Id``) is the only thing we need off the + wire -- it correlates this server-side record with the client's own + record book. No timing values are shipped; each side keeps its own. + """ + server_recv_time = time.monotonic() + request_id = self._request_id(req.headers.get("x-veeksha-request-id")) + record = ServerRequestRecord(request_id, server_recv_time, []) + self.records[request_id] = record + return request_id, record + + async def handle( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + req = await read_http_request(reader) + if req is None: + close_writer(writer) + return + try: + if req.path.startswith("/health"): + await write_response(writer, "200 OK", b"ok") + elif req.path.startswith("/preflight/records"): + payload = json.dumps( + {str(k): v.to_json() for k, v in self.records.items()} + ).encode() + await write_response(writer, "200 OK", payload, "application/json") + elif req.method == "POST": + await self.handle_post(req, writer) + else: + await write_response(writer, "404 Not Found", b"not found") + finally: + close_writer(writer) + + async def handle_post( + self, req: HttpRequest, writer: asyncio.StreamWriter + ) -> None: # pragma: no cover - overridden + raise NotImplementedError + + async def serve_forever(self) -> None: + server = await asyncio.start_server(self.handle, self.host, self.port) + async with server: + await server.serve_forever() diff --git a/veeksha/preflight/servers/base_server.py b/veeksha/preflight/servers/base_server.py new file mode 100644 index 00000000..69bf75d8 --- /dev/null +++ b/veeksha/preflight/servers/base_server.py @@ -0,0 +1,100 @@ +"""Minimal asyncio HTTP/1.1 plumbing shared by the mock servers. + +Deliberately tiny and dependency-free (stdlib only): a real framework would drag +in more scheduling jitter than we can afford when the whole point is to emit on +sub-millisecond deadlines. We parse just enough HTTP to serve the handful of +routes the clients and the preflight harness use. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Dict, Optional + + +@dataclass +class HttpRequest: + method: str + path: str + headers: Dict[str, str] # header names lower-cased + body: bytes + + +async def read_http_request(reader: asyncio.StreamReader) -> Optional[HttpRequest]: + """Read one HTTP/1.1 request. Returns None if the peer closed cleanly.""" + try: + head = await reader.readuntil(b"\r\n\r\n") + except asyncio.IncompleteReadError: + return None + except (ConnectionError, OSError): + return None + + text = head[:-4].decode("latin1") + lines = text.split("\r\n") + parts = lines[0].split(" ") + if len(parts) < 2: + return None + method, path = parts[0], parts[1] + + headers: Dict[str, str] = {} + for line in lines[1:]: + if ":" in line: + k, v = line.split(":", 1) + headers[k.strip().lower()] = v.strip() + + body = b"" + n = int(headers.get("content-length", "0") or "0") + if n > 0: + try: + body = await reader.readexactly(n) + except (asyncio.IncompleteReadError, ConnectionError, OSError): + return None + + return HttpRequest(method=method, path=path, headers=headers, body=body) + + +async def write_response( + writer: asyncio.StreamWriter, + status: str, + body: bytes, + content_type: str = "text/plain", +) -> None: + """Write a complete (non-streaming) response and flush.""" + head = ( + f"HTTP/1.1 {status}\r\n" + f"Content-Type: {content_type}\r\n" + f"Content-Length: {len(body)}\r\n" + "Connection: close\r\n\r\n" + ).encode("latin1") + writer.write(head + body) + try: + await writer.drain() + except (ConnectionError, OSError): + pass + + +async def start_streaming_response( + writer: asyncio.StreamWriter, content_type: str +) -> None: + """Begin a streamed response (no Content-Length; ends at EOF/close).""" + head = ( + "HTTP/1.1 200 OK\r\n" + f"Content-Type: {content_type}\r\n" + "Cache-Control: no-cache\r\n" + "Connection: close\r\n\r\n" + ).encode("latin1") + writer.write(head) + await writer.drain() + + +async def start_sse_response(writer: asyncio.StreamWriter) -> None: + """Begin a Server-Sent-Events stream.""" + await start_streaming_response(writer, "text/event-stream") + + +def close_writer(writer: asyncio.StreamWriter) -> None: + try: + writer.close() + except (ConnectionError, OSError): + pass diff --git a/veeksha/preflight/servers/base_ws_mock.py b/veeksha/preflight/servers/base_ws_mock.py new file mode 100644 index 00000000..a72fdca2 --- /dev/null +++ b/veeksha/preflight/servers/base_ws_mock.py @@ -0,0 +1,85 @@ +"""Shared skeleton for preflight WebSocket mock servers. + +Like base_mock.py but over WebSockets. A single ``websockets`` server both: +- upgrades WS connections (one per request) to ``serve_session`` (overridden per + protocol), stamping t_sr on accept and reading the request id off the + handshake headers; and +- answers plain HTTP ``GET /health`` and ``GET /preflight/records`` via + ``process_request`` so the parent can probe readiness and fetch ground truth. + +No timing values cross the wire -- the request id (handshake header) is the only +thing needed to correlate this server-side record with the client's record book. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from http import HTTPStatus +from typing import Dict, Optional, Tuple + +from websockets.asyncio.server import serve + +from veeksha.preflight.models import ServerRequestRecord + + +class BaseWSMockServer: + def __init__(self, host: str, port: int) -> None: + self.host = host + self.port = port + self.records: Dict[int, ServerRequestRecord] = {} + self._synthetic = -1 + + def _next_synthetic_id(self) -> int: + rid = self._synthetic + self._synthetic -= 1 + return rid + + def _request_id(self, headers) -> int: + value = headers.get("X-Veeksha-Request-Id") + if value is None: + return self._next_synthetic_id() + try: + return int(value) + except ValueError: + return self._next_synthetic_id() + + def _process_request(self, connection, request): + path = request.path + if path.startswith("/health"): + return connection.respond(HTTPStatus.OK, "ok") + if path.startswith("/preflight/records"): + body = json.dumps({str(k): v.to_json() for k, v in self.records.items()}) + return connection.respond(HTTPStatus.OK, body) + return None # proceed with the WebSocket handshake + + def open_record(self, connection) -> Tuple[int, ServerRequestRecord]: + """Stamp receipt (t_sr = connection accept) and start a record.""" + server_recv_time = time.monotonic() + request_id = self._request_id(connection.request.headers) + record = ServerRequestRecord(request_id, server_recv_time, []) + self.records[request_id] = record + return request_id, record + + async def _handler(self, connection) -> None: + _, record = self.open_record(connection) + try: + await self.serve_session(connection, record) + except Exception: + # A client that hangs up mid-stream is normal; the record is kept. + pass + + async def serve_session(self, connection, record: ServerRequestRecord) -> None: + raise NotImplementedError + + async def serve_forever(self) -> None: + async with serve( + self._handler, + self.host, + self.port, + process_request=self._process_request, + max_size=None, + compression=None, + ): + await asyncio.get_running_loop().create_future() # run forever diff --git a/veeksha/preflight/servers/mock_chat_server.py b/veeksha/preflight/servers/mock_chat_server.py new file mode 100644 index 00000000..225a0928 --- /dev/null +++ b/veeksha/preflight/servers/mock_chat_server.py @@ -0,0 +1,98 @@ +"""Deterministic OpenAI-compatible chat mock for preflight validation. + +Emits an SSE stream whose chunks land on absolute deadlines derived from a fixed +``ttfc`` (first-chunk delay) and ``tpoc`` (inter-chunk delay). The server records +its own receive/emit stamps in its record book (no timing on the wire); the +scorer joins them with the client's record book by request id to measure +delivery lag and the server's own pacing fidelity. + +Run standalone (this is how :mod:`veeksha.preflight.spawn` launches it):: + + python -m veeksha.preflight.servers.mock_chat_server \ + --host 127.0.0.1 --port 8123 --ttfc-ms 200 --tpoc-ms 20 --num-chunks 64 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import time +from typing import List, Optional + +from veeksha.preflight.servers.base_mock import BaseMockServer +from veeksha.preflight.servers.base_server import HttpRequest, start_sse_response + + +class MockChatServer(BaseMockServer): + def __init__( + self, + host: str, + port: int, + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + chunk_text: str = "tok ", + ) -> None: + super().__init__(host, port) + self.ttfc_ms = ttfc_ms + self.tpoc_ms = tpoc_ms + self.num_chunks = num_chunks + self.chunk_text = chunk_text + + async def handle_post(self, req: HttpRequest, writer: asyncio.StreamWriter) -> None: + _, record = self.open_record(req) + await start_sse_response(writer) + + first_deadline = record.server_recv_time + self.ttfc_ms / 1000.0 + for i in range(self.num_chunks): + deadline = first_deadline + i * self.tpoc_ms / 1000.0 + slack = deadline - time.monotonic() + if slack > 0: + await asyncio.sleep(slack) + + # t_ss_i: stamp immediately before emitting this chunk, into the + # server's own record book (nothing timing-related on the wire). + record.server_send_times.append(time.monotonic()) + data = {"choices": [{"delta": {"content": self.chunk_text}}]} + writer.write(f"data: {json.dumps(data)}\n\n".encode()) + try: + await writer.drain() + except (ConnectionError, OSError): + return # client hung up; stop emitting + + writer.write(b"data: [DONE]\n\n") + try: + await writer.drain() + except (ConnectionError, OSError): + pass + + +def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Preflight mock chat server") + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--ttfc-ms", type=float, default=200.0) + p.add_argument("--tpoc-ms", type=float, default=20.0) + p.add_argument("--num-chunks", type=int, default=64) + return p.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> None: + args = _parse_args(argv) + print(f"[mock_chat_server] listening on {args.host}:{args.port}", flush=True) + server = MockChatServer( + host=args.host, + port=args.port, + ttfc_ms=args.ttfc_ms, + tpoc_ms=args.tpoc_ms, + num_chunks=args.num_chunks, + ) + try: + asyncio.run(server.serve_forever()) + except KeyboardInterrupt: # pragma: no cover + pass + + +if __name__ == "__main__": + main() diff --git a/veeksha/preflight/servers/mock_completions_server.py b/veeksha/preflight/servers/mock_completions_server.py new file mode 100644 index 00000000..c6712e85 --- /dev/null +++ b/veeksha/preflight/servers/mock_completions_server.py @@ -0,0 +1,73 @@ +"""Deterministic OpenAI-compatible completions mock (non-streaming). + +The completions client does a single request/response, so there is one server +send stamp (``ttfc`` = time to produce the whole response); the mock waits that +long, records the emit time in its record book, then returns one JSON body. No +timing values on the wire, no tpoc. + +Run standalone:: + + python -m veeksha.preflight.servers.mock_completions_server \ + --host 127.0.0.1 --port 8124 --ttfc-ms 150 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import time +from typing import List, Optional + +from veeksha.preflight.servers.base_mock import BaseMockServer +from veeksha.preflight.servers.base_server import HttpRequest, write_response + + +class MockCompletionsServer(BaseMockServer): + def __init__( + self, + host: str, + port: int, + ttfc_ms: float, + completion_text: str = "tok tok tok", + ) -> None: + super().__init__(host, port) + self.ttfc_ms = ttfc_ms + self.completion_text = completion_text + + async def handle_post(self, req: HttpRequest, writer: asyncio.StreamWriter) -> None: + _, record = self.open_record(req) + + deadline = record.server_recv_time + self.ttfc_ms / 1000.0 + slack = deadline - time.monotonic() + if slack > 0: + await asyncio.sleep(slack) + + # single response -> single t_ss stamp, into the server's record book + record.server_send_times.append(time.monotonic()) + body = json.dumps( + {"choices": [{"text": self.completion_text, "index": 0}]} + ).encode() + await write_response(writer, "200 OK", body, "application/json") + + +def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Preflight mock completions server") + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--ttfc-ms", type=float, default=150.0) + return p.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> None: + args = _parse_args(argv) + print(f"[mock_completions_server] listening on {args.host}:{args.port}", flush=True) + server = MockCompletionsServer(host=args.host, port=args.port, ttfc_ms=args.ttfc_ms) + try: + asyncio.run(server.serve_forever()) + except KeyboardInterrupt: # pragma: no cover + pass + + +if __name__ == "__main__": + main() diff --git a/veeksha/preflight/servers/mock_realtime_tts_server.py b/veeksha/preflight/servers/mock_realtime_tts_server.py new file mode 100644 index 00000000..d22382ed --- /dev/null +++ b/veeksha/preflight/servers/mock_realtime_tts_server.py @@ -0,0 +1,122 @@ +"""Deterministic OpenAI-realtime TTS mock (WebSocket) for preflight validation. + +Handshake -> drain the client's text input until ``response.create`` -> emit +``num_chunks`` base64 PCM ``response.output_audio.delta`` events on absolute +deadlines (ttfc first-chunk, tpoc inter-chunk) anchored at connection accept +(t_sr) -> terminal ``response.done``. Emit times (t_ss_i) go in the record book; +nothing timing-related crosses the wire. + +Run standalone:: + + python -m veeksha.preflight.servers.mock_realtime_tts_server \ + --host 127.0.0.1 --port 8130 --ttfc-ms 120 --tpoc-ms 10 --num-chunks 48 +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import time +from typing import List, Optional + +from veeksha.preflight.servers.base_ws_mock import BaseWSMockServer + +_PCM_CHUNK = base64.b64encode(b"\x00" * 640).decode("ascii") # ~20ms @16k mono + + +class MockRealtimeTTSServer(BaseWSMockServer): + def __init__( + self, + host: str, + port: int, + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + sample_rate: int = 24000, + ) -> None: + super().__init__(host, port) + self.ttfc_ms = ttfc_ms + self.tpoc_ms = tpoc_ms + self.num_chunks = num_chunks + self.sample_rate = sample_rate + + async def serve_session(self, connection, record) -> None: + # handshake + await connection.send( + json.dumps( + { + "type": "session.updated", + "session": { + "audio": {"output": {"format": {"rate": self.sample_rate}}} + }, + } + ) + ) + + # drain client input until it asks for a response, recording per-segment + # receipt (t_sr_i) for each text item. + async for raw in connection: + recv_time = time.monotonic() + try: + event = json.loads(raw) + except (json.JSONDecodeError, TypeError, ValueError): + continue + etype = event.get("type") + if etype == "conversation.item.create": + record.input_recv_times.append(recv_time) + elif etype == "response.create": + break + + # anchor the ttfc/tpoc schedule at response start (after the input phase) + record.response_start_time = time.monotonic() + await connection.send(json.dumps({"type": "response.created"})) + + first_deadline = record.response_start_time + self.ttfc_ms / 1000.0 + for i in range(self.num_chunks): + deadline = first_deadline + i * self.tpoc_ms / 1000.0 + slack = deadline - time.monotonic() + if slack > 0: + await asyncio.sleep(slack) + record.server_send_times.append(time.monotonic()) # t_ss_i + await connection.send( + json.dumps({"type": "response.output_audio.delta", "delta": _PCM_CHUNK}) + ) + + await connection.send(json.dumps({"type": "response.output_audio.done"})) + await connection.send( + json.dumps({"type": "response.done", "response": {"status": "completed"}}) + ) + + +def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Preflight mock realtime TTS server") + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--ttfc-ms", type=float, default=120.0) + p.add_argument("--tpoc-ms", type=float, default=10.0) + p.add_argument("--num-chunks", type=int, default=48) + return p.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> None: + args = _parse_args(argv) + print( + f"[mock_realtime_tts_server] listening on {args.host}:{args.port}", flush=True + ) + server = MockRealtimeTTSServer( + host=args.host, + port=args.port, + ttfc_ms=args.ttfc_ms, + tpoc_ms=args.tpoc_ms, + num_chunks=args.num_chunks, + ) + try: + asyncio.run(server.serve_forever()) + except KeyboardInterrupt: # pragma: no cover + pass + + +if __name__ == "__main__": + main() diff --git a/veeksha/preflight/servers/mock_stt_server.py b/veeksha/preflight/servers/mock_stt_server.py new file mode 100644 index 00000000..d5271884 --- /dev/null +++ b/veeksha/preflight/servers/mock_stt_server.py @@ -0,0 +1,111 @@ +"""Deterministic STT mock (WebSocket, vllm_realtime dialect) for preflight. + +Handshake ``session.created`` -> drain the client's ``input_audio_buffer.append`` +chunks (recording per-chunk receipt, t_sr_i) until the final +``input_audio_buffer.commit`` -> emit ``num_chunks`` ``transcription.delta`` +events on absolute deadlines (ttfc/tpoc) anchored at response start -> terminal +``transcription.done``. Emit times (t_ss_i) go in the record book. + +Run standalone:: + + python -m veeksha.preflight.servers.mock_stt_server \ + --host 127.0.0.1 --port 8132 --ttfc-ms 120 --tpoc-ms 10 --num-chunks 32 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import time +from typing import List, Optional + +from veeksha.preflight.servers.base_ws_mock import BaseWSMockServer + + +class MockSTTServer(BaseWSMockServer): + def __init__( + self, + host: str, + port: int, + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + delta_text: str = "word ", + ) -> None: + super().__init__(host, port) + self.ttfc_ms = ttfc_ms + self.tpoc_ms = tpoc_ms + self.num_chunks = num_chunks + self.delta_text = delta_text + + async def serve_session(self, connection, record) -> None: + # vllm_realtime handshake: server announces the session first. + await connection.send(json.dumps({"type": "session.created"})) + + # drain client input (session.update, initial commit, append*, final + # commit), recording per-audio-chunk receipt. + async for raw in connection: + recv_time = time.monotonic() + if isinstance(raw, (bytes, bytearray, memoryview)): + continue + try: + event = json.loads(raw) + except (json.JSONDecodeError, TypeError, ValueError): + continue + etype = event.get("type") + if etype == "input_audio_buffer.append": + record.input_recv_times.append(recv_time) + elif etype == "input_audio_buffer.commit" and event.get("final"): + break + + record.response_start_time = time.monotonic() + first_deadline = record.response_start_time + self.ttfc_ms / 1000.0 + for i in range(self.num_chunks): + deadline = first_deadline + i * self.tpoc_ms / 1000.0 + slack = deadline - time.monotonic() + if slack > 0: + await asyncio.sleep(slack) + record.server_send_times.append(time.monotonic()) # t_ss_i + await connection.send( + json.dumps({"type": "transcription.delta", "delta": self.delta_text}) + ) + + await connection.send( + json.dumps( + { + "type": "transcription.done", + "text": self.delta_text * self.num_chunks, + } + ) + ) + + +def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Preflight mock STT server") + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--ttfc-ms", type=float, default=120.0) + p.add_argument("--tpoc-ms", type=float, default=10.0) + p.add_argument("--num-chunks", type=int, default=32) + return p.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> None: + args = _parse_args(argv) + print(f"[mock_stt_server] listening on {args.host}:{args.port}", flush=True) + server = MockSTTServer( + host=args.host, + port=args.port, + ttfc_ms=args.ttfc_ms, + tpoc_ms=args.tpoc_ms, + num_chunks=args.num_chunks, + ) + try: + asyncio.run(server.serve_forever()) + except KeyboardInterrupt: # pragma: no cover + pass + + +if __name__ == "__main__": + main() diff --git a/veeksha/preflight/servers/mock_tts_server.py b/veeksha/preflight/servers/mock_tts_server.py new file mode 100644 index 00000000..f8a4b2b4 --- /dev/null +++ b/veeksha/preflight/servers/mock_tts_server.py @@ -0,0 +1,94 @@ +"""Deterministic TTS mock (HTTP streaming raw audio) for preflight validation. + +Streams ``num_chunks`` raw-audio chunks on absolute deadlines derived from a +fixed ``ttfc`` (first-chunk / ttfc delay) and ``tpoc`` (inter-chunk delay), +recording its own receive/emit stamps in the record book. The audio bytes carry +no timing; the scorer joins the server record with the client's by request id. + +Emit one chunk of exactly ``chunk_bytes`` per tick so the client's +``aiter_bytes(chunk_size=chunk_bytes)`` reads them 1:1 (the driver sets the +client's chunk size to match). + +Run standalone:: + + python -m veeksha.preflight.servers.mock_tts_server \ + --host 127.0.0.1 --port 8125 --ttfc-ms 120 --tpoc-ms 10 \ + --num-chunks 48 --chunk-bytes 1024 +""" + +from __future__ import annotations + +import argparse +import asyncio +import time +from typing import List, Optional + +from veeksha.preflight.servers.base_mock import BaseMockServer +from veeksha.preflight.servers.base_server import HttpRequest, start_streaming_response + + +class MockTTSServer(BaseMockServer): + def __init__( + self, + host: str, + port: int, + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + chunk_bytes: int = 1024, + ) -> None: + super().__init__(host, port) + self.ttfc_ms = ttfc_ms + self.tpoc_ms = tpoc_ms + self.num_chunks = num_chunks + self._chunk = b"\x00" * chunk_bytes + + async def handle_post(self, req: HttpRequest, writer: asyncio.StreamWriter) -> None: + _, record = self.open_record(req) + await start_streaming_response(writer, "application/octet-stream") + + first_deadline = record.server_recv_time + self.ttfc_ms / 1000.0 + for i in range(self.num_chunks): + deadline = first_deadline + i * self.tpoc_ms / 1000.0 + slack = deadline - time.monotonic() + if slack > 0: + await asyncio.sleep(slack) + + record.server_send_times.append(time.monotonic()) # t_ss_i + writer.write(self._chunk) + try: + await writer.drain() + except (ConnectionError, OSError): + return + + +def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Preflight mock TTS server") + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--ttfc-ms", type=float, default=120.0) + p.add_argument("--tpoc-ms", type=float, default=10.0) + p.add_argument("--num-chunks", type=int, default=48) + p.add_argument("--chunk-bytes", type=int, default=1024) + return p.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> None: + args = _parse_args(argv) + print(f"[mock_tts_server] listening on {args.host}:{args.port}", flush=True) + server = MockTTSServer( + host=args.host, + port=args.port, + ttfc_ms=args.ttfc_ms, + tpoc_ms=args.tpoc_ms, + num_chunks=args.num_chunks, + chunk_bytes=args.chunk_bytes, + ) + try: + asyncio.run(server.serve_forever()) + except KeyboardInterrupt: # pragma: no cover + pass + + +if __name__ == "__main__": + main() diff --git a/veeksha/preflight/servers/mock_vajra_tts_server.py b/veeksha/preflight/servers/mock_vajra_tts_server.py new file mode 100644 index 00000000..9dcb2c76 --- /dev/null +++ b/veeksha/preflight/servers/mock_vajra_tts_server.py @@ -0,0 +1,106 @@ +"""Deterministic Vajra TTS-stream mock (WebSocket, binary PCM) for preflight. + +Drains the client's ``input.text`` segments (recording per-segment receipt, +t_sr_i) until ``input.done``, then emits ``num_chunks`` raw int16-PCM binary +frames on absolute deadlines (ttfc first-frame, tpoc inter-frame) anchored at +response start. Emit times (t_ss_i) go in the record book; nothing timing- +related crosses the wire. + +Run standalone:: + + python -m veeksha.preflight.servers.mock_vajra_tts_server \ + --host 127.0.0.1 --port 8131 --ttfc-ms 120 --tpoc-ms 10 --num-chunks 48 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import time +from typing import List, Optional + +from veeksha.preflight.servers.base_ws_mock import BaseWSMockServer + + +class MockVajraTTSServer(BaseWSMockServer): + def __init__( + self, + host: str, + port: int, + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + sample_rate: int = 24000, + frame_bytes: int = 640, + ) -> None: + super().__init__(host, port) + self.ttfc_ms = ttfc_ms + self.tpoc_ms = tpoc_ms + self.num_chunks = num_chunks + self.sample_rate = sample_rate + self._frame = b"\x00\x00" * (frame_bytes // 2) + + async def serve_session(self, connection, record) -> None: + # drain client input (session.config, input.text*, input.done), recording + # per-segment receipt for each text delta. + async for raw in connection: + recv_time = time.monotonic() + if isinstance(raw, (bytes, bytearray, memoryview)): + continue + try: + event = json.loads(raw) + except (json.JSONDecodeError, TypeError, ValueError): + continue + etype = event.get("type") + if etype == "input.text": + record.input_recv_times.append(recv_time) + elif etype == "input.done": + break + + record.response_start_time = time.monotonic() + await connection.send( + json.dumps({"type": "audio.start", "sample_rate": self.sample_rate}) + ) + + first_deadline = record.response_start_time + self.ttfc_ms / 1000.0 + for i in range(self.num_chunks): + deadline = first_deadline + i * self.tpoc_ms / 1000.0 + slack = deadline - time.monotonic() + if slack > 0: + await asyncio.sleep(slack) + record.server_send_times.append(time.monotonic()) # t_ss_i + await connection.send(self._frame) # binary PCM frame + + await connection.send(json.dumps({"type": "audio.done", "error": False})) + await connection.send(json.dumps({"type": "session.done"})) + + +def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Preflight mock Vajra TTS stream server") + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--ttfc-ms", type=float, default=120.0) + p.add_argument("--tpoc-ms", type=float, default=10.0) + p.add_argument("--num-chunks", type=int, default=48) + return p.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> None: + args = _parse_args(argv) + print(f"[mock_vajra_tts_server] listening on {args.host}:{args.port}", flush=True) + server = MockVajraTTSServer( + host=args.host, + port=args.port, + ttfc_ms=args.ttfc_ms, + tpoc_ms=args.tpoc_ms, + num_chunks=args.num_chunks, + ) + try: + asyncio.run(server.serve_forever()) + except KeyboardInterrupt: # pragma: no cover + pass + + +if __name__ == "__main__": + main() diff --git a/veeksha/preflight/spawn.py b/veeksha/preflight/spawn.py new file mode 100644 index 00000000..b1a37abb --- /dev/null +++ b/veeksha/preflight/spawn.py @@ -0,0 +1,248 @@ +"""Launch and manage mock-server subprocesses for preflight validation. + +Servers run out-of-process (own interpreter, optionally core-pinned) so their +emit schedules stay punctual and don't contend with the veeksha clients. The +parent talks to them only over localhost HTTP: readiness via ``/health`` and +ground-truth stamps via ``/preflight/records``. +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from typing import Dict, Optional + +from veeksha.logger import init_logger +from veeksha.preflight.models import ServerRequestRecord + +logger = init_logger(__name__) + + +def find_free_port(host: str = "127.0.0.1") -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind((host, 0)) + return s.getsockname()[1] + + +class MockServerHandle: + """A running mock-server subprocess.""" + + def __init__(self, proc: subprocess.Popen, host: str, port: int) -> None: + self._proc = proc + self.host = host + self.port = port + + @property + def base_url(self) -> str: + return f"http://{self.host}:{self.port}" + + @property + def api_base(self) -> str: + # Trailing slash: clients append e.g. "chat/completions" to this. + return f"http://{self.host}:{self.port}/v1/" + + def is_alive(self) -> bool: + return self._proc.poll() is None + + def wait_until_ready(self, timeout: float = 10.0) -> None: + deadline = time.monotonic() + timeout + last_err: Optional[Exception] = None + while time.monotonic() < deadline: + if not self.is_alive(): + raise RuntimeError( + f"mock server exited early (code {self._proc.returncode}) " + f"before becoming ready" + ) + try: + with urllib.request.urlopen( + f"{self.base_url}/health", timeout=0.5 + ) as resp: + if resp.status == 200: + return + except (urllib.error.URLError, ConnectionError, OSError) as e: + last_err = e + time.sleep(0.02) + raise TimeoutError( + f"mock server at {self.base_url} not ready within {timeout}s " + f"(last error: {last_err})" + ) + + def fetch_records(self) -> Dict[int, ServerRequestRecord]: + with urllib.request.urlopen( + f"{self.base_url}/preflight/records", timeout=5.0 + ) as resp: + raw = resp.read() + payload = __import__("json").loads(raw) + return {int(k): ServerRequestRecord.from_json(v) for k, v in payload.items()} + + def close(self, timeout: float = 5.0) -> None: + if self._proc.poll() is None: + self._proc.terminate() + try: + self._proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: # pragma: no cover + self._proc.kill() + self._proc.wait(timeout=timeout) + + def __enter__(self) -> "MockServerHandle": + return self + + def __exit__(self, *exc) -> None: + self.close() + + +def spawn_mock_server( + module: str, + server_args: Dict[str, object], + *, + host: str = "127.0.0.1", + port: Optional[int] = None, + ready_timeout: float = 10.0, +) -> MockServerHandle: + """Start a preflight mock server module in a subprocess and wait for /health. + + ``server_args`` are passed as ``--key value`` CLI flags (keys use dashes). + """ + port = port or find_free_port(host) + cmd = [ + sys.executable, + "-m", + module, + "--host", + host, + "--port", + str(port), + ] + for key, value in server_args.items(): + cmd += [f"--{key}", str(value)] + + # Free-threaded child; inherit PYTHONPATH so source layout resolves. + env = dict(os.environ) + env["PYTHON_GIL"] = "0" + + logger.info("Spawning %s on %s:%d", module, host, port) + proc = subprocess.Popen(cmd, env=env) + handle = MockServerHandle(proc, host, port) + try: + handle.wait_until_ready(timeout=ready_timeout) + except Exception: + handle.close() + raise + return handle + + +def spawn_mock_chat_server( + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + host: str = "127.0.0.1", + port: Optional[int] = None, + ready_timeout: float = 10.0, +) -> MockServerHandle: + """Start the streaming-chat (SSE) mock server.""" + return spawn_mock_server( + "veeksha.preflight.servers.mock_chat_server", + {"ttfc-ms": ttfc_ms, "tpoc-ms": tpoc_ms, "num-chunks": num_chunks}, + host=host, + port=port, + ready_timeout=ready_timeout, + ) + + +def spawn_mock_completions_server( + ttfc_ms: float, + host: str = "127.0.0.1", + port: Optional[int] = None, + ready_timeout: float = 10.0, +) -> MockServerHandle: + """Start the non-streaming completions mock server.""" + return spawn_mock_server( + "veeksha.preflight.servers.mock_completions_server", + {"ttfc-ms": ttfc_ms}, + host=host, + port=port, + ready_timeout=ready_timeout, + ) + + +def spawn_mock_tts_server( + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + chunk_bytes: int, + host: str = "127.0.0.1", + port: Optional[int] = None, + ready_timeout: float = 10.0, +) -> MockServerHandle: + """Start the streaming-audio (raw bytes) TTS mock server.""" + return spawn_mock_server( + "veeksha.preflight.servers.mock_tts_server", + { + "ttfc-ms": ttfc_ms, + "tpoc-ms": tpoc_ms, + "num-chunks": num_chunks, + "chunk-bytes": chunk_bytes, + }, + host=host, + port=port, + ready_timeout=ready_timeout, + ) + + +def spawn_mock_realtime_tts_server( + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + host: str = "127.0.0.1", + port: Optional[int] = None, + ready_timeout: float = 10.0, +) -> MockServerHandle: + """Start the OpenAI-realtime TTS mock server (WebSocket).""" + return spawn_mock_server( + "veeksha.preflight.servers.mock_realtime_tts_server", + {"ttfc-ms": ttfc_ms, "tpoc-ms": tpoc_ms, "num-chunks": num_chunks}, + host=host, + port=port, + ready_timeout=ready_timeout, + ) + + +def spawn_mock_vajra_tts_server( + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + host: str = "127.0.0.1", + port: Optional[int] = None, + ready_timeout: float = 10.0, +) -> MockServerHandle: + """Start the Vajra TTS-stream mock server (WebSocket, binary PCM).""" + return spawn_mock_server( + "veeksha.preflight.servers.mock_vajra_tts_server", + {"ttfc-ms": ttfc_ms, "tpoc-ms": tpoc_ms, "num-chunks": num_chunks}, + host=host, + port=port, + ready_timeout=ready_timeout, + ) + + +def spawn_mock_stt_server( + ttfc_ms: float, + tpoc_ms: float, + num_chunks: int, + host: str = "127.0.0.1", + port: Optional[int] = None, + ready_timeout: float = 10.0, +) -> MockServerHandle: + """Start the STT mock server (WebSocket, vllm_realtime dialect).""" + return spawn_mock_server( + "veeksha.preflight.servers.mock_stt_server", + {"ttfc-ms": ttfc_ms, "tpoc-ms": tpoc_ms, "num-chunks": num_chunks}, + host=host, + port=port, + ready_timeout=ready_timeout, + ) diff --git a/veeksha/preflight/validator.py b/veeksha/preflight/validator.py new file mode 100644 index 00000000..217d1312 --- /dev/null +++ b/veeksha/preflight/validator.py @@ -0,0 +1,180 @@ +"""Gate a ScoreReport into a verdict. + +Three outcomes, distinguishing *whose* fault a failure is: + +* ``PASS`` -- every gate passes; the harness measures faithfully. +* ``FAIL`` -- the harness itself adds drift beyond threshold + (delivery lag, dispatch drift, or too many unpaired + requests). The benchmark numbers cannot be trusted. +* ``SERVER_AT_CAPACITY`` -- the *mock server* could not hold its own ttfc/tpoc + schedule, i.e. the server was the bottleneck, so the + harness cannot be certified either way (not the + harness's fault; re-run lighter / on a quieter box). + +Server-pacing gates are checked first: if the ground-truth generator is jittery, +the delivery numbers built on top of it are meaningless, so that outcome wins. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import List + +from veeksha.preflight import scorer +from veeksha.preflight.models import ScoreReport + +VERDICT_PASS = "PASS" +VERDICT_FAIL = "FAIL" +VERDICT_SERVER_AT_CAPACITY = "SERVER_AT_CAPACITY" + +# Gate categories: "server" gates blame the mock; "harness" gates blame veeksha. +_CATEGORY_SERVER = "server" +_CATEGORY_HARNESS = "harness" + + +@dataclass +class GateResult: + name: str + metric: str + category: str + p99: float + threshold: float + count: int + passed: bool + + +@dataclass +class ValidationResult: + verdict: str + gates: List[GateResult] = field(default_factory=list) + + @property + def is_pass(self) -> bool: + return self.verdict == VERDICT_PASS + + def failed_gates(self, category: str) -> List[GateResult]: + return [g for g in self.gates if g.category == category and not g.passed] + + +def _gate( + report: ScoreReport, + name: str, + metric: str, + category: str, + threshold: float, + optional: bool = False, +) -> GateResult: + summary = report.metrics.get(metric) + p99 = summary.p99 if summary is not None else float("nan") + count = summary.count if summary is not None else 0 + if optional and count == 0: + # Metric not applicable to this client (e.g. input metrics on a + # non-streaming-input client) -> N/A, treated as passing. + return GateResult(name, metric, category, p99, threshold, count, True) + # A NaN p99 (no data) fails safe: NaN < threshold is False. + passed = (not math.isnan(p99)) and p99 < threshold + return GateResult(name, metric, category, p99, threshold, count, passed) + + +def run_validation( + report: ScoreReport, + *, + delivery_lag_threshold_ms: float, + server_pacing_threshold_ms: float, + dispatch_drift_threshold_ms: float, + input_pacing_threshold_ms: float, + max_unpaired_fraction: float, +) -> ValidationResult: + """Apply the gates and derive the verdict.""" + gates: List[GateResult] = [ + # server pacing fidelity (blames the mock) + _gate( + report, + "server ttfc pacing", + scorer.M_SERVER_TTFC_ABS_ERR, + _CATEGORY_SERVER, + server_pacing_threshold_ms, + ), + _gate( + report, + "server tpoc pacing", + scorer.M_SERVER_TPOC_ABS_ERR, + _CATEGORY_SERVER, + server_pacing_threshold_ms, + ), + # request/response delivery lag (blames the harness transport) + _gate( + report, + "request delivery", + scorer.M_REQUEST_DELIVERY, + _CATEGORY_HARNESS, + delivery_lag_threshold_ms, + ), + _gate( + report, + "response delivery", + scorer.M_RESPONSE_DELIVERY, + _CATEGORY_HARNESS, + delivery_lag_threshold_ms, + ), + # end-to-end dispatch drift (blames the scheduler/dispatcher) + _gate( + report, + "dispatch drift", + scorer.M_LIFECYCLE_READY_TO_SEND, + _CATEGORY_HARNESS, + dispatch_drift_threshold_ms, + ), + # streaming-input only (optional -> N/A for single-shot clients) + _gate( + report, + "input delivery", + scorer.M_INPUT_DELIVERY, + _CATEGORY_HARNESS, + delivery_lag_threshold_ms, + optional=True, + ), + _gate( + report, + "input pacing", + scorer.M_INPUT_PACING_ABS_ERR, + _CATEGORY_HARNESS, + input_pacing_threshold_ms, + optional=True, + ), + ] + + # unpaired-fraction gate (harness dropped or mismatched requests) + unpaired_ok = report.unpaired_fraction <= max_unpaired_fraction + gates.append( + GateResult( + name="unpaired fraction", + metric="unpaired_fraction", + category=_CATEGORY_HARNESS, + p99=report.unpaired_fraction, + threshold=max_unpaired_fraction, + count=report.n_requests, + passed=unpaired_ok, + ) + ) + + # SERVER_AT_CAPACITY only applies when the server actually produced pacing + # data that breached -- a missing (count 0) server gate is not the mock + # being jittery, it means the run itself failed to measure, which is a + # harness problem (falls through to FAIL below). + server_failed = any( + not g.passed and g.count > 0 and g.category == _CATEGORY_SERVER for g in gates + ) + harness_failed = any( + not g.passed and g.category == _CATEGORY_HARNESS for g in gates + ) + + if server_failed: + verdict = VERDICT_SERVER_AT_CAPACITY + elif harness_failed: + verdict = VERDICT_FAIL + else: + verdict = VERDICT_PASS + + return ValidationResult(verdict=verdict, gates=gates) diff --git a/veeksha/traffic/base.py b/veeksha/traffic/base.py index 345dbded..36334ae8 100644 --- a/veeksha/traffic/base.py +++ b/veeksha/traffic/base.py @@ -25,26 +25,22 @@ def schedule_session(self, session: Session) -> None: raise NotImplementedError @abstractmethod - def pop_ready(self) -> Optional[Tuple[Request, int, int]]: - """Pop a ready request from the scheduler. + def pop_ready(self) -> Optional[Tuple[Request, int, int, float]]: + """Pop a ready request. - Returns: - Tuple of (request, session_id, session_size) if a request is ready, - None otherwise. + Returns (request, session_id, session_size, scheduler_ready_at), or None. + ``scheduler_ready_at`` is the scheduled ready instant in the + ``time.monotonic()`` domain. """ raise NotImplementedError @abstractmethod def wait_for_ready( self, timeout: float = 0.001 - ) -> Optional[Tuple[Request, int, int]]: - """Wait for a ready request with timeout. + ) -> Optional[Tuple[Request, int, int, float]]: + """Wait up to ``timeout`` seconds for a ready request. - Args: - timeout: Maximum time to wait in seconds. - - Returns: - Tuple of (request, session_id, session_size) if ready, None if timeout. + Same return shape as :meth:`pop_ready`; None on timeout. """ raise NotImplementedError diff --git a/veeksha/traffic/concurrent.py b/veeksha/traffic/concurrent.py index ed23bfaa..28b9a470 100644 --- a/veeksha/traffic/concurrent.py +++ b/veeksha/traffic/concurrent.py @@ -98,14 +98,15 @@ def schedule_session(self, session: Session) -> None: def wait_for_ready( self, timeout: float = 0.001 - ) -> Optional[Tuple[Request, int, int]]: + ) -> Optional[Tuple[Request, int, int, float]]: """Wait for a ready request with timeout. Args: timeout: Maximum time to wait in seconds. Returns: - Tuple of (request, session_id, session_size) if ready, None if timeout. + Tuple of (request, session_id, session_size, scheduler_ready_at) if + ready, None if timeout. """ with self._condition: result = self._try_pop_ready_locked() @@ -123,7 +124,7 @@ def wait_for_ready( # check again after waking return self._try_pop_ready_locked() - def _try_pop_ready_locked(self) -> Optional[Tuple[Request, int, int]]: + def _try_pop_ready_locked(self) -> Optional[Tuple[Request, int, int, float]]: """Try to pop a ready item, must be called with lock held.""" if not self._ready_queue: return None @@ -137,10 +138,12 @@ def _try_pop_ready_locked(self) -> Optional[Tuple[Request, int, int]]: session_size = len(state.session.requests) self._populate_history(request, state, node_id) - return (request, session_id, session_size) + # ready_at (offset from scheduler start) as an absolute monotonic time. + scheduler_ready_at = item.ready_at + self._start_monotonic + return (request, session_id, session_size, scheduler_ready_at) return None - def pop_ready(self) -> Optional[Tuple[Request, int, int]]: + def pop_ready(self) -> Optional[Tuple[Request, int, int, float]]: with self._condition: return self._try_pop_ready_locked() diff --git a/veeksha/traffic/rate.py b/veeksha/traffic/rate.py index 46fb838f..830ba921 100644 --- a/veeksha/traffic/rate.py +++ b/veeksha/traffic/rate.py @@ -78,14 +78,15 @@ def schedule_session(self, session: Session) -> None: def wait_for_ready( self, timeout: float = 0.001 - ) -> Optional[Tuple[Request, int, int]]: + ) -> Optional[Tuple[Request, int, int, float]]: """Wait for a ready request with timeout. Args: timeout: Maximum time to wait in seconds. Returns: - Tuple of (request, session_id, session_size) if ready, None if timeout. + Tuple of (request, session_id, session_size, scheduler_ready_at) if + ready, None if timeout. """ with self._condition: result = self._try_pop_ready_locked() @@ -101,21 +102,24 @@ def wait_for_ready( self._condition.wait(timeout=wait_time) return self._try_pop_ready_locked() - def _try_pop_ready_locked(self) -> Optional[Tuple[Request, int, int]]: + def _try_pop_ready_locked(self) -> Optional[Tuple[Request, int, int, float]]: """Try to pop a ready item, must be called with lock held.""" if not self._ready_queue: return None if self._ready_queue[0].ready_at <= self._now(): - request = heapq.heappop(self._ready_queue).request + item = heapq.heappop(self._ready_queue) + request = item.request session_id, node_id = self._request_to_session[request.id] state = self._sessions[session_id] session_size = len(state.session.requests) self._populate_history(request, state, node_id) - return (request, session_id, session_size) + # ready_at (offset from scheduler start) as an absolute monotonic time. + scheduler_ready_at = item.ready_at + self._start_monotonic + return (request, session_id, session_size, scheduler_ready_at) return None - def pop_ready(self) -> Optional[Tuple[Request, int, int]]: + def pop_ready(self) -> Optional[Tuple[Request, int, int, float]]: with self._condition: return self._try_pop_ready_locked() diff --git a/veeksha/traffic/sequential_launch.py b/veeksha/traffic/sequential_launch.py index 83fc7da1..349dfe5e 100644 --- a/veeksha/traffic/sequential_launch.py +++ b/veeksha/traffic/sequential_launch.py @@ -92,7 +92,7 @@ def schedule_session(self, session: Session) -> None: def wait_for_ready( self, timeout: float = 0.001 - ) -> Optional[tuple[Request, int, int]]: + ) -> Optional[tuple[Request, int, int, float]]: with self._condition: result = self._try_pop_ready_locked() if result is not None: @@ -107,7 +107,7 @@ def wait_for_ready( self._condition.wait(timeout=wait_time) return self._try_pop_ready_locked() - def _try_pop_ready_locked(self) -> Optional[tuple[Request, int, int]]: + def _try_pop_ready_locked(self) -> Optional[tuple[Request, int, int, float]]: if not self._ready_queue: return None @@ -121,10 +121,12 @@ def _try_pop_ready_locked(self) -> Optional[tuple[Request, int, int]]: self._populate_history(request, state, node_id) - return (request, session_id, session_size) + # ready_at (offset from scheduler start) as an absolute monotonic time. + scheduler_ready_at = item.ready_at + self._start_monotonic + return (request, session_id, session_size, scheduler_ready_at) return None - def pop_ready(self) -> Optional[tuple[Request, int, int]]: + def pop_ready(self) -> Optional[tuple[Request, int, int, float]]: with self._condition: return self._try_pop_ready_locked() diff --git a/veeksha/workers/client_runner.py b/veeksha/workers/client_runner.py index 5b8bec6c..79dd8c43 100644 --- a/veeksha/workers/client_runner.py +++ b/veeksha/workers/client_runner.py @@ -92,6 +92,9 @@ async def _run_async(self) -> None: task.cancel() await asyncio.wait(active_tasks, timeout=2.0) + # Sessions bind to this loop, which ``run()`` closes next. + await self.client.aclose() + logger.debug("Client worker %d exiting", self.worker_id) async def _process_request(self, item) -> None: diff --git a/veeksha/workers/dispatch.py b/veeksha/workers/dispatch.py index edee7283..1ce8ff14 100644 --- a/veeksha/workers/dispatch.py +++ b/veeksha/workers/dispatch.py @@ -73,8 +73,9 @@ def run(self) -> None: if result is None: continue - request, session_id, session_size = result - scheduler_ready_at = time.monotonic() + request, session_id, session_size, scheduler_ready_at = result + # scheduler_ready_at: scheduled ready time; dispatched_at: actual + # dequeue time. dispatched_at = time.monotonic() self.evaluator.register_request( @@ -121,8 +122,7 @@ def _drain(self) -> None: if result is None: break - request, session_id, session_size = result - scheduler_ready_at = time.monotonic() + request, session_id, session_size, scheduler_ready_at = result dispatched_at = time.monotonic() self.evaluator.register_request(