From 301433b5ff13575b8913b5e23cee72f615bebc24 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Tue, 26 May 2026 23:28:13 +0100 Subject: [PATCH 1/3] Add sample-centered CLI inspection --- .../ergon_cli/domains/experiments/commands.py | 108 ++++++++++++++---- .../ergon_cli/domains/experiments/models.py | 64 ++++++++++- .../ergon_cli/domains/experiments/parser.py | 9 +- .../ergon_cli/domains/experiments/service.py | 89 ++++++++++++++- .../ergon_cli/domains/samples/commands.py | 67 ++++++++++- ergon_cli/ergon_cli/domains/samples/models.py | 39 +++++++ ergon_cli/ergon_cli/domains/samples/parser.py | 6 + .../ergon_cli/domains/samples/service.py | 61 +++++++++- .../unit/cli/test_cli_state_rendering.py | 80 +++++++++++++ .../tests/unit/cli/test_experiment_cli.py | 47 +++++--- .../cli/test_no_generic_experiment_submit.py | 20 ++++ ergon_cli/tests/unit/cli/test_sample_cli.py | 6 + 12 files changed, 548 insertions(+), 48 deletions(-) create mode 100644 ergon_cli/tests/unit/cli/test_cli_state_rendering.py create mode 100644 ergon_cli/tests/unit/cli/test_no_generic_experiment_submit.py diff --git a/ergon_cli/ergon_cli/domains/experiments/commands.py b/ergon_cli/ergon_cli/domains/experiments/commands.py index f9f742b6e..f3facce1c 100644 --- a/ergon_cli/ergon_cli/domains/experiments/commands.py +++ b/ergon_cli/ergon_cli/domains/experiments/commands.py @@ -1,6 +1,8 @@ from argparse import Namespace from ergon_cli.domains.experiments.models import ( + ExperimentSamplesCommand, + ExperimentSamplerInvocationsCommand, ListByTagCommand, ListExperimentsCommand, ListTagsCommand, @@ -8,6 +10,8 @@ ) from ergon_cli.domains.experiments.service import ( list_by_tag, + list_experiment_samples, + list_experiment_sampler_invocations, list_experiments, list_tags, show_experiment, @@ -23,11 +27,15 @@ async def handle_experiment(args: Namespace) -> int: return handle_experiment_show(args) if args.experiment_action == "list": return handle_experiment_list(args) + if args.experiment_action == "samples": + return handle_experiment_samples(args) + if args.experiment_action == "sampler-invocations": + return handle_experiment_sampler_invocations(args) if args.experiment_action == "tags": return handle_experiment_tags(args) if args.experiment_action == "by-tag": return handle_experiment_by_tag(args) - print("Usage: ergon experiment {show|list|tags|by-tag}") + print("Usage: ergon experiment {show|list|samples|sampler-invocations|tags|by-tag}") return exit_codes.RUNTIME_ERROR @@ -35,40 +43,98 @@ def handle_experiment_show(args: Namespace) -> int: try: detail = show_experiment( ShowExperimentCommand( - definition_id=parse_uuid(args.definition_id, field_name="definition_id") + experiment_id=parse_uuid(args.experiment_id, field_name="experiment_id") ) ) except CliError as exc: print(exc.message) return exc.exit_code - experiment = detail.experiment lines = [ - f"DEFINITION_ID={experiment.definition_id}", - f"NAME={experiment.name}", - f"BENCHMARK={experiment.benchmark_type}", - f"STATUS={experiment.status}", - f"SAMPLE_COUNT={experiment.sample_count}", - f"RUN_COUNT={experiment.run_count}", + f"EXPERIMENT_ID={detail.experiment_id}", + f"NAME={detail.name}", + f"SAMPLE_COUNT={detail.sample_count}", + f"SAMPLER_INVOCATIONS={detail.sampler_invocation_count}", ] - if experiment.default_model_target is not None: - lines.append(f"DEFAULT_MODEL={experiment.default_model_target}") - if experiment.default_evaluator_slug is not None: - lines.append(f"DEFAULT_EVALUATOR={experiment.default_evaluator_slug}") - if detail.sample_selection: - lines.append(f"SAMPLE_SELECTION={detail.sample_selection}") - if detail.runs: + if detail.environments: + lines.append("ENVIRONMENTS") + lines.extend( + "\t".join( + [ + environment.environment_name, + environment.source_mode, + str(environment.sample_count), + str(environment.selected_count), + ] + ) + for environment in detail.environments + ) + if detail.samples: lines.append("SAMPLES") lines.extend( "\t".join( [ - str(run.sample_id), - run.instance_key, - run.status, - "" if run.model_target is None else run.model_target, + str(sample.sample_id), + sample.environment_name, + sample.sample_key, + sample.status, ] ) - for run in detail.runs + for sample in detail.samples + ) + print(render_text(lines)) + return exit_codes.OK + + +def handle_experiment_samples(args: Namespace) -> int: + try: + result = list_experiment_samples( + ExperimentSamplesCommand( + experiment_id=parse_uuid(args.experiment_id, field_name="experiment_id") + ) + ) + except CliError as exc: + print(exc.message) + return exc.exit_code + if not result.samples: + print("No samples found.") + return exit_codes.OK + lines = ["SAMPLE_ID\tENVIRONMENT\tKEY\tSTATUS"] + lines.extend( + "\t".join( + [str(sample.sample_id), sample.environment_name, sample.sample_key, sample.status] + ) + for sample in result.samples + ) + print(render_text(lines)) + return exit_codes.OK + + +def handle_experiment_sampler_invocations(args: Namespace) -> int: + try: + result = list_experiment_sampler_invocations( + ExperimentSamplerInvocationsCommand( + experiment_id=parse_uuid(args.experiment_id, field_name="experiment_id") + ) ) + except CliError as exc: + print(exc.message) + return exc.exit_code + if not result.invocations: + print("No sampler invocations found.") + return exit_codes.OK + lines = ["SAMPLER_INVOCATION_ID\tSAMPLER\tK\tCANDIDATES\tSELECTED"] + lines.extend( + "\t".join( + [ + str(invocation.sampler_invocation_id), + invocation.sampler_name, + str(invocation.requested_k), + str(invocation.candidate_pool_size), + str(invocation.selected_count), + ] + ) + for invocation in result.invocations + ) print(render_text(lines)) return exit_codes.OK diff --git a/ergon_cli/ergon_cli/domains/experiments/models.py b/ergon_cli/ergon_cli/domains/experiments/models.py index e7dc7aff2..d03e1dbd2 100644 --- a/ergon_cli/ergon_cli/domains/experiments/models.py +++ b/ergon_cli/ergon_cli/domains/experiments/models.py @@ -10,7 +10,17 @@ class ListExperimentsCommand(BaseModel): class ShowExperimentCommand(BaseModel): model_config = ConfigDict(frozen=True) - definition_id: UUID + experiment_id: UUID + + +class ExperimentSamplesCommand(BaseModel): + model_config = ConfigDict(frozen=True) + experiment_id: UUID + + +class ExperimentSamplerInvocationsCommand(BaseModel): + model_config = ConfigDict(frozen=True) + experiment_id: UUID class ListTagsCommand(BaseModel): @@ -59,3 +69,55 @@ class ExperimentTagDefinitionView(BaseModel): name: str benchmark_type: str latest_run_status: str | None = None + + +class ExperimentEnvironmentCliState(BaseModel): + model_config = ConfigDict(frozen=True) + + environment_id: UUID + environment_name: str + source_mode: str + sample_count: int + selected_count: int + + +class ExperimentSampleCliState(BaseModel): + model_config = ConfigDict(frozen=True) + + sample_id: UUID + environment_name: str + sample_key: str + status: str + + +class SamplerInvocationCliState(BaseModel): + model_config = ConfigDict(frozen=True) + + sampler_invocation_id: UUID + sampler_name: str + requested_k: int + candidate_pool_size: int + selected_count: int + + +class ExperimentCliState(BaseModel): + model_config = ConfigDict(frozen=True) + + experiment_id: UUID + name: str + environments: tuple[ExperimentEnvironmentCliState, ...] + sample_count: int + sampler_invocation_count: int + samples: tuple[ExperimentSampleCliState, ...] = () + + +class ExperimentSamplesCliResult(BaseModel): + model_config = ConfigDict(frozen=True) + + samples: tuple[ExperimentSampleCliState, ...] + + +class ExperimentSamplerInvocationsCliResult(BaseModel): + model_config = ConfigDict(frozen=True) + + invocations: tuple[SamplerInvocationCliState, ...] diff --git a/ergon_cli/ergon_cli/domains/experiments/parser.py b/ergon_cli/ergon_cli/domains/experiments/parser.py index 247ad5e8d..93c61cbaf 100644 --- a/ergon_cli/ergon_cli/domains/experiments/parser.py +++ b/ergon_cli/ergon_cli/domains/experiments/parser.py @@ -8,9 +8,16 @@ def register_experiment_parser(subparsers: argparse._SubParsersAction) -> None: experiment.set_defaults(handler=handle_experiment) experiment_sub = experiment.add_subparsers(dest="experiment_action") experiment_show = experiment_sub.add_parser("show", help="Show experiment detail") - experiment_show.add_argument("definition_id", help="Experiment UUID") + experiment_show.add_argument("experiment_id", help="Experiment UUID") experiment_list = experiment_sub.add_parser("list", help="List experiments") experiment_list.add_argument("--limit", type=int, default=50, help="Number of experiments") + experiment_samples = experiment_sub.add_parser("samples", help="List samples in an experiment") + experiment_samples.add_argument("experiment_id", help="Experiment UUID") + experiment_invocations = experiment_sub.add_parser( + "sampler-invocations", + help="List sampler invocations for an experiment", + ) + experiment_invocations.add_argument("experiment_id", help="Experiment UUID") experiment_sub.add_parser("tags", help="List distinct experiment tags") experiment_by_tag = experiment_sub.add_parser( "by-tag", help="List definitions for an experiment tag" diff --git a/ergon_cli/ergon_cli/domains/experiments/service.py b/ergon_cli/ergon_cli/domains/experiments/service.py index e747383ed..f490d6164 100644 --- a/ergon_cli/ergon_cli/domains/experiments/service.py +++ b/ergon_cli/ergon_cli/domains/experiments/service.py @@ -1,15 +1,29 @@ from ergon_cli.domains.experiments.models import ( + ExperimentCliState, ExperimentDetailView, + ExperimentEnvironmentCliState, ExperimentRunView, + ExperimentSampleCliState, + ExperimentSamplesCliResult, + ExperimentSamplesCommand, + ExperimentSamplerInvocationsCliResult, + ExperimentSamplerInvocationsCommand, ExperimentSummaryView, ExperimentTagDefinitionView, ListByTagCommand, ListExperimentsCommand, ListTagsCommand, + SamplerInvocationCliState, ShowExperimentCommand, ) from ergon_cli.shared.errors import CliNotFoundError -from ergon_core.core.views.experiments.models import ExperimentDetailDto, ExperimentSummaryDto +from ergon_core.core.views.experiments.models import ( + ExperimentDetailDto, + ExperimentDetailView as CoreExperimentDetailView, + ExperimentSampleSummaryView, + ExperimentSummaryDto, + SamplerInvocationView, +) from ergon_core.core.views.experiments.service import ExperimentReadService @@ -26,12 +40,38 @@ def show_experiment( command: ShowExperimentCommand, *, read_service: ExperimentReadService | None = None, -) -> ExperimentDetailView: +) -> ExperimentCliState: service = read_service or ExperimentReadService() - detail = service.get_experiment(command.definition_id) + detail = service.get_experiment_state(command.experiment_id) if detail is None: - raise CliNotFoundError(f"Experiment not found: {command.definition_id}") - return _detail_view(detail) + raise CliNotFoundError(f"Experiment not found: {command.experiment_id}") + return _experiment_state_view(detail) + + +def list_experiment_samples( + command: ExperimentSamplesCommand, + *, + read_service: ExperimentReadService | None = None, +) -> ExperimentSamplesCliResult: + service = read_service or ExperimentReadService() + result = service.list_experiment_samples(command.experiment_id) + if result is None: + raise CliNotFoundError(f"Experiment not found: {command.experiment_id}") + return ExperimentSamplesCliResult(samples=tuple(_sample_view(row) for row in result.items)) + + +def list_experiment_sampler_invocations( + command: ExperimentSamplerInvocationsCommand, + *, + read_service: ExperimentReadService | None = None, +) -> ExperimentSamplerInvocationsCliResult: + service = read_service or ExperimentReadService() + result = service.list_sampler_invocations(command.experiment_id) + if result is None: + raise CliNotFoundError(f"Experiment not found: {command.experiment_id}") + return ExperimentSamplerInvocationsCliResult( + invocations=tuple(_sampler_invocation_view(row) for row in result.items) + ) def list_tags( @@ -88,3 +128,42 @@ def _detail_view(detail: ExperimentDetailDto) -> ExperimentDetailView: for run in detail.runs ), ) + + +def _experiment_state_view(detail: CoreExperimentDetailView) -> ExperimentCliState: + return ExperimentCliState( + experiment_id=detail.experiment_id, + name=detail.name, + environments=tuple( + ExperimentEnvironmentCliState( + environment_id=environment.environment_id, + environment_name=environment.environment_name, + source_mode=environment.source_mode, + sample_count=environment.sample_count, + selected_count=environment.selected_count, + ) + for environment in detail.environments + ), + sample_count=detail.sample_count, + sampler_invocation_count=len(detail.sampler_invocations), + samples=tuple(_sample_view(row) for row in detail.samples), + ) + + +def _sample_view(row: ExperimentSampleSummaryView) -> ExperimentSampleCliState: + return ExperimentSampleCliState( + sample_id=row.sample_id, + environment_name=row.environment_name, + sample_key=row.sample_key, + status=row.status, + ) + + +def _sampler_invocation_view(row: SamplerInvocationView) -> SamplerInvocationCliState: + return SamplerInvocationCliState( + sampler_invocation_id=row.sampler_invocation_id, + sampler_name=row.sampler_name, + requested_k=row.requested_k, + candidate_pool_size=row.candidate_pool_size, + selected_count=row.selected_count, + ) diff --git a/ergon_cli/ergon_cli/domains/samples/commands.py b/ergon_cli/ergon_cli/domains/samples/commands.py index cebd4b847..4f5251a84 100644 --- a/ergon_cli/ergon_cli/domains/samples/commands.py +++ b/ergon_cli/ergon_cli/domains/samples/commands.py @@ -3,11 +3,16 @@ from ergon_cli.domains.samples.models import ( CancelSampleCommand, ListSamplesCommand, + SampleEventsCommand, + SampleGraphCommand, SampleStatusCommand, ) from ergon_cli.domains.samples.service import ( cancel_existing_sample, + get_sample_detail, + get_sample_graph, get_sample_status, + list_sample_events, list_samples, ) from ergon_cli.shared import exit_codes @@ -23,7 +28,13 @@ def handle_sample(args: Namespace) -> int: return cancel_sample_command(args) if args.sample_action == "status": return status_sample_command(args) - print("Usage: ergon sample {list|status|cancel}") + if args.sample_action == "show": + return show_sample_command(args) + if args.sample_action == "events": + return sample_events_command(args) + if args.sample_action == "graph": + return sample_graph_command(args) + print("Usage: ergon sample {list|status|show|events|graph|cancel}") return exit_codes.RUNTIME_ERROR @@ -98,7 +109,6 @@ def status_sample_command(args: Namespace) -> int: f"sample_id: {sample.id}", f"status: {sample.status}", f"benchmark_type: {sample.benchmark_type}", - f"definition_id: {sample.definition_id}", f"instance_key: {sample.instance_key}", ] if sample.evaluator_slug is not None: @@ -114,3 +124,56 @@ def status_sample_command(args: Namespace) -> int: lines.append(f"error: {sample.error_message}") print(render_text(lines)) return exit_codes.OK + + +def show_sample_command(args: Namespace) -> int: + try: + sample = get_sample_detail(SampleStatusCommand(sample_id=parse_uuid(args.sample_id))) + except CliError as exc: + print(exc.message) + return exc.exit_code + print( + render_text( + [ + f"sample_id: {sample.sample_id}", + f"experiment_id: {sample.experiment_id}", + f"environment_id: {sample.environment_id}", + f"environment: {sample.environment_name}", + f"sample_key: {sample.sample_key}", + f"status: {sample.status}", + ] + ) + ) + return exit_codes.OK + + +def sample_events_command(args: Namespace) -> int: + try: + events = list_sample_events(SampleEventsCommand(sample_id=parse_uuid(args.sample_id))) + except CliError as exc: + print(exc.message) + return exc.exit_code + if not events: + print("No events found.") + return exit_codes.OK + lines = ["EVENT\tTARGET\tTIMESTAMP"] + lines.extend("\t".join([event.event_type, event.target, event.timestamp]) for event in events) + print(render_text(lines)) + return exit_codes.OK + + +def sample_graph_command(args: Namespace) -> int: + try: + graph = get_sample_graph(SampleGraphCommand(sample_id=parse_uuid(args.sample_id))) + except CliError as exc: + print(exc.message) + return exc.exit_code + lines = [ + f"nodes: {graph.node_count}", + f"edges: {graph.edge_count}", + ] + if graph.nodes: + lines.append("NODES") + lines.extend(graph.nodes) + print(render_text(lines)) + return exit_codes.OK diff --git a/ergon_cli/ergon_cli/domains/samples/models.py b/ergon_cli/ergon_cli/domains/samples/models.py index 235551ca5..3ed5c3830 100644 --- a/ergon_cli/ergon_cli/domains/samples/models.py +++ b/ergon_cli/ergon_cli/domains/samples/models.py @@ -18,6 +18,18 @@ class SampleStatusCommand(BaseModel): sample_id: UUID +class SampleEventsCommand(BaseModel): + model_config = ConfigDict(frozen=True) + + sample_id: UUID + + +class SampleGraphCommand(BaseModel): + model_config = ConfigDict(frozen=True) + + sample_id: UUID + + class CancelSampleCommand(BaseModel): model_config = ConfigDict(frozen=True) @@ -41,6 +53,33 @@ class SampleSummaryView(BaseModel): error_message: str | None = None +class SampleDetailCliState(BaseModel): + model_config = ConfigDict(frozen=True) + + sample_id: UUID + experiment_id: UUID + environment_id: UUID + environment_name: str + sample_key: str + status: str + + +class SampleEventCliState(BaseModel): + model_config = ConfigDict(frozen=True) + + event_type: str + target: str + timestamp: str + + +class SampleGraphCliState(BaseModel): + model_config = ConfigDict(frozen=True) + + node_count: int + edge_count: int + nodes: tuple[str, ...] + + class SampleListResult(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/ergon_cli/ergon_cli/domains/samples/parser.py b/ergon_cli/ergon_cli/domains/samples/parser.py index d190ef054..f76f96484 100644 --- a/ergon_cli/ergon_cli/domains/samples/parser.py +++ b/ergon_cli/ergon_cli/domains/samples/parser.py @@ -30,3 +30,9 @@ def register_sample_parser(subparsers: argparse._SubParsersAction) -> None: sample_cancel_parser.add_argument("sample_id", help="Sample ID (UUID) to cancel") sample_status_parser = sample_sub.add_parser("status", help="Show status of one sample") sample_status_parser.add_argument("sample_id", help="Sample ID (UUID)") + sample_show_parser = sample_sub.add_parser("show", help="Show sample detail") + sample_show_parser.add_argument("sample_id", help="Sample ID (UUID)") + sample_events_parser = sample_sub.add_parser("events", help="List typed WAL events for a sample") + sample_events_parser.add_argument("sample_id", help="Sample ID (UUID)") + sample_graph_parser = sample_sub.add_parser("graph", help="Show sample graph projection") + sample_graph_parser.add_argument("sample_id", help="Sample ID (UUID)") diff --git a/ergon_cli/ergon_cli/domains/samples/service.py b/ergon_cli/ergon_cli/domains/samples/service.py index 0f22d4c05..0b5bf2fd4 100644 --- a/ergon_cli/ergon_cli/domains/samples/service.py +++ b/ergon_cli/ergon_cli/domains/samples/service.py @@ -3,13 +3,18 @@ CancelSampleResult, ListSamplesCommand, SampleListResult, + SampleDetailCliState, + SampleEventCliState, + SampleEventsCommand, + SampleGraphCliState, + SampleGraphCommand, SampleStatusCommand, SampleSummaryView, ) from ergon_cli.shared.errors import CliNotFoundError from ergon_core.core.application.runtime.sample_records import cancel_sample from ergon_core.core.views.samples.models import SampleSummaryDto -from ergon_core.core.views.samples.service import SampleSnapshotReadService +from ergon_core.core.views.samples.service import SampleReadService, SampleSnapshotReadService def list_samples( @@ -42,6 +47,60 @@ def get_sample_status( return _view(row) +def get_sample_detail( + command: SampleStatusCommand, + *, + read_service: SampleReadService | None = None, +) -> SampleDetailCliState: + service = read_service or SampleReadService() + row = service.get_sample_detail(command.sample_id) + if row is None: + raise CliNotFoundError(f"No sample found with id {command.sample_id}") + return SampleDetailCliState( + sample_id=row.sample_id, + experiment_id=row.experiment_id, + environment_id=row.environment_id, + environment_name=row.environment_name, + sample_key=row.sample_key, + status=row.status, + ) + + +def list_sample_events( + command: SampleEventsCommand, + *, + read_service: SampleReadService | None = None, +) -> tuple[SampleEventCliState, ...]: + service = read_service or SampleReadService() + events = service.list_sample_events(command.sample_id) + if events is None: + raise CliNotFoundError(f"No sample found with id {command.sample_id}") + return tuple( + SampleEventCliState( + event_type=event.event_type, + target=event.target_type, + timestamp=event.timestamp.isoformat(), + ) + for event in events.items + ) + + +def get_sample_graph( + command: SampleGraphCommand, + *, + read_service: SampleReadService | None = None, +) -> SampleGraphCliState: + service = read_service or SampleReadService() + graph = service.get_sample_graph(command.sample_id) + if graph is None: + raise CliNotFoundError(f"No sample found with id {command.sample_id}") + return SampleGraphCliState( + node_count=len(graph.nodes), + edge_count=len(graph.edges), + nodes=tuple(f"{node.task_slug}\t{node.status}" for node in graph.nodes), + ) + + def cancel_existing_sample(command: CancelSampleCommand) -> CancelSampleResult: try: sample = cancel_sample(command.sample_id) diff --git a/ergon_cli/tests/unit/cli/test_cli_state_rendering.py b/ergon_cli/tests/unit/cli/test_cli_state_rendering.py new file mode 100644 index 000000000..038d5aa80 --- /dev/null +++ b/ergon_cli/tests/unit/cli/test_cli_state_rendering.py @@ -0,0 +1,80 @@ +from uuid import uuid4 + +from ergon_cli.domains.experiments.commands import handle_experiment_show +from ergon_cli.domains.experiments.models import ( + ExperimentCliState, + ExperimentEnvironmentCliState, +) +from ergon_cli.domains.samples.commands import sample_events_command +from ergon_cli.domains.samples.models import SampleEventCliState +from argparse import Namespace + + +def test_experiment_state_rendering_uses_environment_and_sample_vocabulary( + monkeypatch, + capsys, +) -> None: + experiment_id = uuid4() + + def fake_show(command): + return ExperimentCliState( + experiment_id=command.experiment_id, + name="mixed-training", + environments=( + ExperimentEnvironmentCliState( + environment_id=uuid4(), + environment_name="mini-validation", + source_mode="materialized", + sample_count=3, + selected_count=3, + ), + ), + sample_count=3, + sampler_invocation_count=1, + ) + + monkeypatch.setattr("ergon_cli.domains.experiments.commands.show_experiment", fake_show) + + rc = handle_experiment_show(Namespace(experiment_id=str(experiment_id))) + + assert rc == 0 + text = capsys.readouterr().out + assert "mixed-training" in text + assert "mini-validation" in text + assert "SAMPLE_COUNT" in text + assert "RUN_COUNT" not in text + assert "DEFINITION" not in text + + +def test_sample_event_rendering_uses_typed_wal_events(monkeypatch, capsys) -> None: + sample_id = uuid4() + + def fake_events(command): + return ( + SampleEventCliState( + event_type="sample.status_changed", + target="sample", + timestamp="2026-05-26T00:00:00Z", + ), + SampleEventCliState( + event_type="task.added", + target="task", + timestamp="2026-05-26T00:00:01Z", + ), + SampleEventCliState( + event_type="worker.added", + target="task", + timestamp="2026-05-26T00:00:02Z", + ), + ) + + monkeypatch.setattr("ergon_cli.domains.samples.commands.list_sample_events", fake_events) + + rc = sample_events_command(Namespace(sample_id=str(sample_id))) + + assert rc == 0 + text = capsys.readouterr().out + assert "sample.status_changed" in text + assert "task.added" in text + assert "worker.added" in text + assert "GraphMutation" not in text diff --git a/ergon_cli/tests/unit/cli/test_experiment_cli.py b/ergon_cli/tests/unit/cli/test_experiment_cli.py index b67a4d3ac..6d6fabb29 100644 --- a/ergon_cli/tests/unit/cli/test_experiment_cli.py +++ b/ergon_cli/tests/unit/cli/test_experiment_cli.py @@ -7,9 +7,12 @@ from ergon_cli.domains.experiments.models import ExperimentTagDefinitionView from ergon_cli.main import build_parser from ergon_core.core.views.experiments.models import ( + EnvironmentContributionView, ExperimentDetailDto, + ExperimentDetailView as CoreExperimentDetailView, ExperimentRunMetricsDto, ExperimentRunRowDto, + ExperimentSampleSummaryView, ExperimentSummaryDto, ) @@ -92,39 +95,49 @@ def list_experiments(self, *, limit: int): def test_experiment_show_prints_detail(monkeypatch, capsys): sample_id = uuid4() + experiment_id = uuid4() + environment_id = uuid4() class FakeReadService: - def get_experiment(self, definition_id): - return ExperimentDetailDto( - experiment=_summary(definition_id=definition_id), - runs=[ - ExperimentRunRowDto( + def get_experiment_state(self, requested_experiment_id): + assert requested_experiment_id == experiment_id + return CoreExperimentDetailView( + experiment_id=experiment_id, + name="ci experiment", + environments=[ + EnvironmentContributionView( + environment_id=environment_id, + environment_name="mini-validation", + source_mode="materialized", + sample_count=1, + selected_count=1, + ) + ], + sample_count=1, + samples=[ + ExperimentSampleSummaryView( sample_id=sample_id, - definition_id=uuid4(), - benchmark_type="ci-benchmark", - instance_key="sample-a", + experiment_id=experiment_id, + environment_id=environment_id, + environment_name="mini-validation", + sample_key="sample-a", status="completed", created_at="2026-04-27T12:00:00Z", - metrics=ExperimentRunMetricsDto( - sample_id=sample_id, - status="completed", - instance_key="sample-a", - ), ) ], - sample_selection={"instance_keys": ["sample-a"]}, + created_at="2026-04-27T12:00:00Z", ) - definition_id = uuid4() monkeypatch.setattr(experiment_domain_service, "ExperimentReadService", FakeReadService) - rc = experiment_cmd.handle_experiment_show(Namespace(definition_id=str(definition_id))) + rc = experiment_cmd.handle_experiment_show(Namespace(experiment_id=str(experiment_id))) assert rc == 0 out = capsys.readouterr().out - assert str(definition_id) in out + assert str(experiment_id) in out assert str(sample_id) in out assert "sample-a" in out + assert "definition" not in out.lower() def test_experiment_tag_subcommands_are_registered() -> None: diff --git a/ergon_cli/tests/unit/cli/test_no_generic_experiment_submit.py b/ergon_cli/tests/unit/cli/test_no_generic_experiment_submit.py new file mode 100644 index 000000000..c55b18ca3 --- /dev/null +++ b/ergon_cli/tests/unit/cli/test_no_generic_experiment_submit.py @@ -0,0 +1,20 @@ +from uuid import uuid4 + +import pytest + +from ergon_cli.main import build_parser + + +@pytest.mark.parametrize( + "argv", + [ + ["experiment", "persist", "path/to/experiment.py"], + ["experiment", "submit", str(uuid4())], + ["experiment", "submit", str(uuid4()), "--k", "32"], + ], +) +def test_cli_does_not_offer_generic_experiment_submit_or_persist(argv) -> None: + parser = build_parser() + + with pytest.raises(SystemExit): + parser.parse_args(argv) diff --git a/ergon_cli/tests/unit/cli/test_sample_cli.py b/ergon_cli/tests/unit/cli/test_sample_cli.py index 8a74d7fbc..01b013eb7 100644 --- a/ergon_cli/tests/unit/cli/test_sample_cli.py +++ b/ergon_cli/tests/unit/cli/test_sample_cli.py @@ -25,10 +25,16 @@ def test_sample_subcommands_are_registered_in_main_parser() -> None: parser = build_parser() status_args = parser.parse_args(["sample", "status", str(uuid4())]) + show_args = parser.parse_args(["sample", "show", str(uuid4())]) + events_args = parser.parse_args(["sample", "events", str(uuid4())]) + graph_args = parser.parse_args(["sample", "graph", str(uuid4())]) definition_id = uuid4() list_args = parser.parse_args(["sample", "list", "--definition-id", str(definition_id)]) assert status_args.sample_action == "status" + assert show_args.sample_action == "show" + assert events_args.sample_action == "events" + assert graph_args.sample_action == "graph" assert list_args.sample_action == "list" assert list_args.definition_id == str(definition_id) From 5c657615d3e1201794ff582b8f98dc032a7d6ff9 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 27 May 2026 00:18:04 +0100 Subject: [PATCH 2/3] Allow CLI sample identity surfaces --- ergon_cli/ergon_cli/domains/samples/parser.py | 4 +++- .../unit/architecture/test_definition_identity_naming.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ergon_cli/ergon_cli/domains/samples/parser.py b/ergon_cli/ergon_cli/domains/samples/parser.py index f76f96484..c045a1516 100644 --- a/ergon_cli/ergon_cli/domains/samples/parser.py +++ b/ergon_cli/ergon_cli/domains/samples/parser.py @@ -32,7 +32,9 @@ def register_sample_parser(subparsers: argparse._SubParsersAction) -> None: sample_status_parser.add_argument("sample_id", help="Sample ID (UUID)") sample_show_parser = sample_sub.add_parser("show", help="Show sample detail") sample_show_parser.add_argument("sample_id", help="Sample ID (UUID)") - sample_events_parser = sample_sub.add_parser("events", help="List typed WAL events for a sample") + sample_events_parser = sample_sub.add_parser( + "events", help="List typed WAL events for a sample" + ) sample_events_parser.add_argument("sample_id", help="Sample ID (UUID)") sample_graph_parser = sample_sub.add_parser("graph", help="Show sample graph projection") sample_graph_parser.add_argument("sample_id", help="Sample ID (UUID)") diff --git a/ergon_core/tests/unit/architecture/test_definition_identity_naming.py b/ergon_core/tests/unit/architecture/test_definition_identity_naming.py index ea917c163..c0653ec83 100644 --- a/ergon_core/tests/unit/architecture/test_definition_identity_naming.py +++ b/ergon_core/tests/unit/architecture/test_definition_identity_naming.py @@ -30,6 +30,9 @@ ROOT / "ergon_core" / "tests" / "unit" / "core" / "application" / "experiments", ROOT / "ergon_core" / "tests" / "unit" / "read_models", ROOT / "ergon_core" / "tests" / "unit" / "rest_api", + ROOT / "ergon_cli" / "ergon_cli" / "domains" / "experiments", + ROOT / "ergon_cli" / "ergon_cli" / "domains" / "samples", + ROOT / "ergon_cli" / "tests" / "unit" / "cli", ROOT / "tests" / "examples", ) ALLOWED_EXPERIMENT_ID_FILES = { From a2a8ddf0730e3ce49748a7d8eb65d9b9350b4140 Mon Sep 17 00:00:00 2001 From: Charlie Masters <69640669+cm2435@users.noreply.github.com> Date: Wed, 27 May 2026 13:59:50 +0100 Subject: [PATCH 3/3] Make CLI sample inspection only --- .../ergon_cli/domains/experiments/commands.py | 56 ++----------- .../ergon_cli/domains/experiments/models.py | 54 ++----------- .../ergon_cli/domains/experiments/parser.py | 5 -- .../ergon_cli/domains/experiments/service.py | 72 ++--------------- .../ergon_cli/domains/samples/commands.py | 33 +------- ergon_cli/ergon_cli/domains/samples/models.py | 15 ---- ergon_cli/ergon_cli/domains/samples/parser.py | 7 -- .../ergon_cli/domains/samples/service.py | 16 ---- .../tests/unit/cli/test_experiment_cli.py | 78 ++++--------------- ergon_cli/tests/unit/cli/test_sample_cli.py | 52 +++---------- 10 files changed, 47 insertions(+), 341 deletions(-) diff --git a/ergon_cli/ergon_cli/domains/experiments/commands.py b/ergon_cli/ergon_cli/domains/experiments/commands.py index f3facce1c..b5d8037c0 100644 --- a/ergon_cli/ergon_cli/domains/experiments/commands.py +++ b/ergon_cli/ergon_cli/domains/experiments/commands.py @@ -3,17 +3,13 @@ from ergon_cli.domains.experiments.models import ( ExperimentSamplesCommand, ExperimentSamplerInvocationsCommand, - ListByTagCommand, ListExperimentsCommand, - ListTagsCommand, ShowExperimentCommand, ) from ergon_cli.domains.experiments.service import ( - list_by_tag, list_experiment_samples, list_experiment_sampler_invocations, list_experiments, - list_tags, show_experiment, ) from ergon_cli.shared import exit_codes @@ -31,11 +27,7 @@ async def handle_experiment(args: Namespace) -> int: return handle_experiment_samples(args) if args.experiment_action == "sampler-invocations": return handle_experiment_sampler_invocations(args) - if args.experiment_action == "tags": - return handle_experiment_tags(args) - if args.experiment_action == "by-tag": - return handle_experiment_by_tag(args) - print("Usage: ergon experiment {show|list|samples|sampler-invocations|tags|by-tag}") + print("Usage: ergon experiment {show|list|samples|sampler-invocations}") return exit_codes.RUNTIME_ERROR @@ -140,55 +132,21 @@ def handle_experiment_sampler_invocations(args: Namespace) -> int: def handle_experiment_list(args: Namespace) -> int: - experiments = list_experiments(ListExperimentsCommand(limit=args.limit)) - if not experiments: + result = list_experiments(ListExperimentsCommand(limit=args.limit)) + if not result.experiments: print("No experiments found.") return exit_codes.OK - lines = ["DEFINITION_ID\tNAME\tBENCHMARK\tSTATUS\tSAMPLES\tATTEMPTS\tMODEL"] + lines = ["EXPERIMENT_ID\tNAME\tSAMPLES\tSAMPLER_INVOCATIONS"] lines.extend( "\t".join( [ - str(experiment.definition_id), + str(experiment.experiment_id), experiment.name, - experiment.benchmark_type, - experiment.status, str(experiment.sample_count), - str(experiment.run_count), - "" if experiment.default_model_target is None else experiment.default_model_target, + str(experiment.sampler_invocation_count), ] ) - for experiment in experiments - ) - print(render_text(lines)) - return exit_codes.OK - - -def handle_experiment_tags(args: Namespace) -> int: - del args - tags = list_tags(ListTagsCommand()) - if not tags: - print("No experiment tags found.") - return exit_codes.OK - print(render_text(tags)) - return exit_codes.OK - - -def handle_experiment_by_tag(args: Namespace) -> int: - rows = list_by_tag(ListByTagCommand(tag=args.tag)) - if not rows: - print(f"No definitions found for experiment tag {args.tag!r}.") - return exit_codes.OK - lines = ["DEFINITION_ID\tNAME\tBENCHMARK\tLATEST_RUN_STATUS"] - lines.extend( - "\t".join( - [ - str(row.definition_id), - row.name, - row.benchmark_type, - "" if row.latest_run_status is None else row.latest_run_status, - ] - ) - for row in rows + for experiment in result.experiments ) print(render_text(lines)) return exit_codes.OK diff --git a/ergon_cli/ergon_cli/domains/experiments/models.py b/ergon_cli/ergon_cli/domains/experiments/models.py index d03e1dbd2..8236dc85f 100644 --- a/ergon_cli/ergon_cli/domains/experiments/models.py +++ b/ergon_cli/ergon_cli/domains/experiments/models.py @@ -23,54 +23,6 @@ class ExperimentSamplerInvocationsCommand(BaseModel): experiment_id: UUID -class ListTagsCommand(BaseModel): - model_config = ConfigDict(frozen=True) - - -class ListByTagCommand(BaseModel): - model_config = ConfigDict(frozen=True) - tag: str - - -class ExperimentSummaryView(BaseModel): - model_config = ConfigDict(frozen=True) - - definition_id: UUID - name: str - benchmark_type: str - status: str - sample_count: int - run_count: int - default_model_target: str | None = None - default_evaluator_slug: str | None = None - - -class ExperimentRunView(BaseModel): - model_config = ConfigDict(frozen=True) - - sample_id: UUID - instance_key: str - status: str - model_target: str | None = None - - -class ExperimentDetailView(BaseModel): - model_config = ConfigDict(frozen=True) - - experiment: ExperimentSummaryView - sample_selection: dict - runs: tuple[ExperimentRunView, ...] - - -class ExperimentTagDefinitionView(BaseModel): - model_config = ConfigDict(frozen=True) - - definition_id: UUID - name: str - benchmark_type: str - latest_run_status: str | None = None - - class ExperimentEnvironmentCliState(BaseModel): model_config = ConfigDict(frozen=True) @@ -111,6 +63,12 @@ class ExperimentCliState(BaseModel): samples: tuple[ExperimentSampleCliState, ...] = () +class ExperimentListCliResult(BaseModel): + model_config = ConfigDict(frozen=True) + + experiments: tuple[ExperimentCliState, ...] + + class ExperimentSamplesCliResult(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/ergon_cli/ergon_cli/domains/experiments/parser.py b/ergon_cli/ergon_cli/domains/experiments/parser.py index 93c61cbaf..d63292ed3 100644 --- a/ergon_cli/ergon_cli/domains/experiments/parser.py +++ b/ergon_cli/ergon_cli/domains/experiments/parser.py @@ -18,8 +18,3 @@ def register_experiment_parser(subparsers: argparse._SubParsersAction) -> None: help="List sampler invocations for an experiment", ) experiment_invocations.add_argument("experiment_id", help="Experiment UUID") - experiment_sub.add_parser("tags", help="List distinct experiment tags") - experiment_by_tag = experiment_sub.add_parser( - "by-tag", help="List definitions for an experiment tag" - ) - experiment_by_tag.add_argument("tag", help="Experiment tag") diff --git a/ergon_cli/ergon_cli/domains/experiments/service.py b/ergon_cli/ergon_cli/domains/experiments/service.py index f490d6164..9826152cb 100644 --- a/ergon_cli/ergon_cli/domains/experiments/service.py +++ b/ergon_cli/ergon_cli/domains/experiments/service.py @@ -1,27 +1,20 @@ from ergon_cli.domains.experiments.models import ( ExperimentCliState, - ExperimentDetailView, ExperimentEnvironmentCliState, - ExperimentRunView, + ExperimentListCliResult, ExperimentSampleCliState, ExperimentSamplesCliResult, ExperimentSamplesCommand, ExperimentSamplerInvocationsCliResult, ExperimentSamplerInvocationsCommand, - ExperimentSummaryView, - ExperimentTagDefinitionView, - ListByTagCommand, ListExperimentsCommand, - ListTagsCommand, SamplerInvocationCliState, ShowExperimentCommand, ) from ergon_cli.shared.errors import CliNotFoundError from ergon_core.core.views.experiments.models import ( - ExperimentDetailDto, ExperimentDetailView as CoreExperimentDetailView, ExperimentSampleSummaryView, - ExperimentSummaryDto, SamplerInvocationView, ) from ergon_core.core.views.experiments.service import ExperimentReadService @@ -31,9 +24,12 @@ def list_experiments( command: ListExperimentsCommand, *, read_service: ExperimentReadService | None = None, -) -> list[ExperimentSummaryView]: +) -> ExperimentListCliResult: service = read_service or ExperimentReadService() - return [_summary_view(row) for row in service.list_experiments(limit=command.limit)] + result = service.list_experiment_states(limit=command.limit) + return ExperimentListCliResult( + experiments=tuple(_experiment_state_view(row) for row in result.items) + ) def show_experiment( @@ -74,62 +70,6 @@ def list_experiment_sampler_invocations( ) -def list_tags( - command: ListTagsCommand, - *, - read_service: ExperimentReadService | None = None, -) -> list[str]: - del command - service = read_service or ExperimentReadService() - return service.distinct_tags() - - -def list_by_tag( - command: ListByTagCommand, - *, - read_service: ExperimentReadService | None = None, -) -> list[ExperimentTagDefinitionView]: - service = read_service or ExperimentReadService() - return [ - ExperimentTagDefinitionView( - definition_id=row.definition_id, - name=row.name, - benchmark_type=row.benchmark_type, - latest_run_status=row.latest_run_status, - ) - for row in service.definitions_by_tag(command.tag) - ] - - -def _summary_view(row: ExperimentSummaryDto) -> ExperimentSummaryView: - return ExperimentSummaryView( - definition_id=row.definition_id, - name=row.name, - benchmark_type=row.benchmark_type, - status=row.status, - sample_count=row.sample_count, - run_count=row.run_count, - default_model_target=row.default_model_target, - default_evaluator_slug=row.default_evaluator_slug, - ) - - -def _detail_view(detail: ExperimentDetailDto) -> ExperimentDetailView: - return ExperimentDetailView( - experiment=_summary_view(detail.experiment), - sample_selection=detail.sample_selection, - runs=tuple( - ExperimentRunView( - sample_id=run.sample_id, - instance_key=run.instance_key, - status=run.status, - model_target=run.model_target, - ) - for run in detail.runs - ), - ) - - def _experiment_state_view(detail: CoreExperimentDetailView) -> ExperimentCliState: return ExperimentCliState( experiment_id=detail.experiment_id, diff --git a/ergon_cli/ergon_cli/domains/samples/commands.py b/ergon_cli/ergon_cli/domains/samples/commands.py index 4f5251a84..8681ef63b 100644 --- a/ergon_cli/ergon_cli/domains/samples/commands.py +++ b/ergon_cli/ergon_cli/domains/samples/commands.py @@ -1,14 +1,12 @@ from argparse import Namespace from ergon_cli.domains.samples.models import ( - CancelSampleCommand, ListSamplesCommand, SampleEventsCommand, SampleGraphCommand, SampleStatusCommand, ) from ergon_cli.domains.samples.service import ( - cancel_existing_sample, get_sample_detail, get_sample_graph, get_sample_status, @@ -24,8 +22,6 @@ def handle_sample(args: Namespace) -> int: if args.sample_action == "list": return list_samples_command(args) - if args.sample_action == "cancel": - return cancel_sample_command(args) if args.sample_action == "status": return status_sample_command(args) if args.sample_action == "show": @@ -34,22 +30,16 @@ def handle_sample(args: Namespace) -> int: return sample_events_command(args) if args.sample_action == "graph": return sample_graph_command(args) - print("Usage: ergon sample {list|status|show|events|graph|cancel}") + print("Usage: ergon sample {list|status|show|events|graph}") return exit_codes.RUNTIME_ERROR def list_samples_command(args: Namespace) -> int: try: - definition_id = ( - parse_uuid(args.definition_id, field_name="definition_id") - if args.definition_id - else None - ) result = list_samples( ListSamplesCommand( limit=args.limit, status=args.status, - definition_id=definition_id, experiment=args.experiment, ) ) @@ -61,8 +51,6 @@ def list_samples_command(args: Namespace) -> int: parts = ["No samples found"] if result.status: parts.append(f"with status={result.status!r}") - if result.definition_id: - parts.append(f"for definition_id={str(result.definition_id)!r}") if result.experiment: parts.append(f"for experiment={result.experiment!r}") print(" ".join(parts)) @@ -80,25 +68,6 @@ def list_samples_command(args: Namespace) -> int: return exit_codes.OK -def cancel_sample_command(args: Namespace) -> int: - try: - result = cancel_existing_sample(CancelSampleCommand(sample_id=parse_uuid(args.sample_id))) - except CliError as exc: - print(exc.message) - return exc.exit_code - print( - render_text( - [ - f"Sample {result.sample.id} cancelled.", - f" Status: {result.sample.status}", - " Inngest: sample/cancelled event sent (in-flight functions will be killed)", - " Cleanup: sample/cleanup event sent (sandbox teardown scheduled)", - ] - ) - ) - return exit_codes.OK - - def status_sample_command(args: Namespace) -> int: try: sample = get_sample_status(SampleStatusCommand(sample_id=parse_uuid(args.sample_id))) diff --git a/ergon_cli/ergon_cli/domains/samples/models.py b/ergon_cli/ergon_cli/domains/samples/models.py index 3ed5c3830..2445891f2 100644 --- a/ergon_cli/ergon_cli/domains/samples/models.py +++ b/ergon_cli/ergon_cli/domains/samples/models.py @@ -8,7 +8,6 @@ class ListSamplesCommand(BaseModel): limit: int = 20 status: str | None = None - definition_id: UUID | None = None experiment: str | None = None @@ -30,12 +29,6 @@ class SampleGraphCommand(BaseModel): sample_id: UUID -class CancelSampleCommand(BaseModel): - model_config = ConfigDict(frozen=True) - - sample_id: UUID - - class SampleSummaryView(BaseModel): model_config = ConfigDict(frozen=True) @@ -45,7 +38,6 @@ class SampleSummaryView(BaseModel): started: str | None = None completed: str | None = None duration: str - definition_id: UUID benchmark_type: str instance_key: str evaluator_slug: str | None = None @@ -85,11 +77,4 @@ class SampleListResult(BaseModel): samples: tuple[SampleSummaryView, ...] status: str | None = None - definition_id: UUID | None = None experiment: str | None = None - - -class CancelSampleResult(BaseModel): - model_config = ConfigDict(frozen=True) - - sample: SampleSummaryView diff --git a/ergon_cli/ergon_cli/domains/samples/parser.py b/ergon_cli/ergon_cli/domains/samples/parser.py index c045a1516..1ded3b2ef 100644 --- a/ergon_cli/ergon_cli/domains/samples/parser.py +++ b/ergon_cli/ergon_cli/domains/samples/parser.py @@ -16,18 +16,11 @@ def register_sample_parser(subparsers: argparse._SubParsersAction) -> None: default=None, help="Filter by status (pending, executing, completed, failed, cancelled)", ) - sample_list_parser.add_argument( - "--definition-id", - default=None, - help="Filter by definition UUID", - ) sample_list_parser.add_argument( "--experiment", default=None, help="Filter by v2 experiment tag", ) - sample_cancel_parser = sample_sub.add_parser("cancel", help="Cancel a running sample") - sample_cancel_parser.add_argument("sample_id", help="Sample ID (UUID) to cancel") sample_status_parser = sample_sub.add_parser("status", help="Show status of one sample") sample_status_parser.add_argument("sample_id", help="Sample ID (UUID)") sample_show_parser = sample_sub.add_parser("show", help="Show sample detail") diff --git a/ergon_cli/ergon_cli/domains/samples/service.py b/ergon_cli/ergon_cli/domains/samples/service.py index 0b5bf2fd4..caeed7ffa 100644 --- a/ergon_cli/ergon_cli/domains/samples/service.py +++ b/ergon_cli/ergon_cli/domains/samples/service.py @@ -1,6 +1,4 @@ from ergon_cli.domains.samples.models import ( - CancelSampleCommand, - CancelSampleResult, ListSamplesCommand, SampleListResult, SampleDetailCliState, @@ -12,7 +10,6 @@ SampleSummaryView, ) from ergon_cli.shared.errors import CliNotFoundError -from ergon_core.core.application.runtime.sample_records import cancel_sample from ergon_core.core.views.samples.models import SampleSummaryDto from ergon_core.core.views.samples.service import SampleReadService, SampleSnapshotReadService @@ -24,13 +21,11 @@ def list_samples( rows = service.list_samples( limit=command.limit, status=command.status, - definition_id=command.definition_id, experiment=command.experiment, ) return SampleListResult( samples=tuple(_view(row) for row in rows), status=command.status, - definition_id=command.definition_id, experiment=command.experiment, ) @@ -101,16 +96,6 @@ def get_sample_graph( ) -def cancel_existing_sample(command: CancelSampleCommand) -> CancelSampleResult: - try: - sample = cancel_sample(command.sample_id) - except ValueError as exc: - raise CliNotFoundError(f"Error: {exc}") from exc - return CancelSampleResult( - sample=_view(SampleSummaryDto.model_validate(sample, from_attributes=True)) - ) - - def _view(row: SampleSummaryDto) -> SampleSummaryView: created = row.created_at.strftime("%Y-%m-%d %H:%M") if row.created_at else "-" started = row.started_at.strftime("%Y-%m-%d %H:%M:%S") if row.started_at else None @@ -126,7 +111,6 @@ def _view(row: SampleSummaryDto) -> SampleSummaryView: started=started, completed=completed, duration=duration, - definition_id=row.definition_id, benchmark_type=row.benchmark_type, instance_key=row.instance_key, evaluator_slug=row.evaluator_slug, diff --git a/ergon_cli/tests/unit/cli/test_experiment_cli.py b/ergon_cli/tests/unit/cli/test_experiment_cli.py index 6d6fabb29..177e3f650 100644 --- a/ergon_cli/tests/unit/cli/test_experiment_cli.py +++ b/ergon_cli/tests/unit/cli/test_experiment_cli.py @@ -4,31 +4,24 @@ import pytest import ergon_cli.domains.experiments.commands as experiment_cmd import ergon_cli.domains.experiments.service as experiment_domain_service -from ergon_cli.domains.experiments.models import ExperimentTagDefinitionView from ergon_cli.main import build_parser from ergon_core.core.views.experiments.models import ( EnvironmentContributionView, - ExperimentDetailDto, ExperimentDetailView as CoreExperimentDetailView, - ExperimentRunMetricsDto, - ExperimentRunRowDto, ExperimentSampleSummaryView, - ExperimentSummaryDto, + ExperimentListView, ) -def _summary(**overrides) -> ExperimentSummaryDto: +def _experiment_state(**overrides) -> CoreExperimentDetailView: data = { - "definition_id": uuid4(), + "experiment_id": uuid4(), "name": "ci experiment", - "benchmark_type": "ci-benchmark", "sample_count": 2, - "status": "defined", "created_at": "2026-04-27T12:00:00Z", - "run_count": 0, } data.update(overrides) - return ExperimentSummaryDto.model_validate(data) + return CoreExperimentDetailView.model_validate(data) def test_experiment_subcommands_are_registered_in_main_parser() -> None: @@ -78,9 +71,14 @@ def test_experiment_run_subcommand_is_no_longer_registered() -> None: def test_experiment_list_prints_rows(monkeypatch, capsys): class FakeReadService: - def list_experiments(self, *, limit: int): + def list_experiment_states(self, *, limit: int): assert limit == 3 - return [_summary(name="alpha"), _summary(name="beta", status="running", run_count=2)] + return ExperimentListView( + items=[ + _experiment_state(name="alpha", sample_count=1), + _experiment_state(name="beta", sample_count=2), + ] + ) monkeypatch.setattr(experiment_domain_service, "ExperimentReadService", FakeReadService) @@ -90,7 +88,7 @@ def list_experiments(self, *, limit: int): out = capsys.readouterr().out assert "alpha" in out assert "beta" in out - assert "running" in out + assert "EXPERIMENT_ID" in out def test_experiment_show_prints_detail(monkeypatch, capsys): @@ -140,53 +138,11 @@ def get_experiment_state(self, requested_experiment_id): assert "definition" not in out.lower() -def test_experiment_tag_subcommands_are_registered() -> None: +def test_experiment_tag_subcommands_are_not_registered() -> None: parser = build_parser() - tags_args = parser.parse_args(["experiment", "tags"]) - by_tag_args = parser.parse_args(["experiment", "by-tag", "alpha"]) - - assert tags_args.experiment_action == "tags" - assert by_tag_args.experiment_action == "by-tag" - assert by_tag_args.tag == "alpha" - - -def test_experiment_tags_prints_run_record_experiment_tags(monkeypatch, capsys): - class FakeTagService: - def distinct_tags(self): - return ["alpha", "beta"] - - monkeypatch.setattr(experiment_domain_service, "ExperimentReadService", FakeTagService) - - rc = experiment_cmd.handle_experiment_tags(Namespace()) - - assert rc == 0 - out = capsys.readouterr().out - assert "alpha" in out - assert "beta" in out - - -def test_experiment_by_tag_prints_definitions_for_run_record_tag(monkeypatch, capsys): - definition_id = uuid4() - - class FakeTagService: - def definitions_by_tag(self, tag): - assert tag == "alpha" - return [ - ExperimentTagDefinitionView( - definition_id=definition_id, - name="alpha definition", - benchmark_type="ci-benchmark", - latest_run_status="completed", - ) - ] - - monkeypatch.setattr(experiment_domain_service, "ExperimentReadService", FakeTagService) - - rc = experiment_cmd.handle_experiment_by_tag(Namespace(tag="alpha")) + with pytest.raises(SystemExit): + parser.parse_args(["experiment", "tags"]) - assert rc == 0 - out = capsys.readouterr().out - assert str(definition_id) in out - assert "alpha definition" in out - assert "completed" in out + with pytest.raises(SystemExit): + parser.parse_args(["experiment", "by-tag", "alpha"]) diff --git a/ergon_cli/tests/unit/cli/test_sample_cli.py b/ergon_cli/tests/unit/cli/test_sample_cli.py index 01b013eb7..0d4586aa1 100644 --- a/ergon_cli/tests/unit/cli/test_sample_cli.py +++ b/ergon_cli/tests/unit/cli/test_sample_cli.py @@ -28,15 +28,14 @@ def test_sample_subcommands_are_registered_in_main_parser() -> None: show_args = parser.parse_args(["sample", "show", str(uuid4())]) events_args = parser.parse_args(["sample", "events", str(uuid4())]) graph_args = parser.parse_args(["sample", "graph", str(uuid4())]) - definition_id = uuid4() - list_args = parser.parse_args(["sample", "list", "--definition-id", str(definition_id)]) + list_args = parser.parse_args(["sample", "list", "--limit", "3"]) assert status_args.sample_action == "status" assert show_args.sample_action == "show" assert events_args.sample_action == "events" assert graph_args.sample_action == "graph" assert list_args.sample_action == "list" - assert list_args.definition_id == str(definition_id) + assert list_args.limit == 3 def test_sample_list_accepts_experiment_tag_filter() -> None: @@ -48,6 +47,13 @@ def test_sample_list_accepts_experiment_tag_filter() -> None: assert list_args.experiment == "alpha" +def test_sample_cancel_is_not_registered() -> None: + parser = build_parser() + + with pytest.raises(SystemExit): + parser.parse_args(["sample", "cancel", str(uuid4())]) + + # --------------------------------------------------------------------------- # SQLite session fixture (mirrors Task 1 pattern) # --------------------------------------------------------------------------- @@ -189,42 +195,6 @@ def get(self, model, pk): # --------------------------------------------------------------------------- -# test_sample_list_filters_by_definition -# --------------------------------------------------------------------------- - - -def test_sample_list_filters_by_definition(monkeypatch, session_factory, capsys): - """Only samples for the requested definition appear.""" - definition_matching = _definition(name="matching") - definition_other = _definition(name="other") - - sample_matching = _sample_record(definition_id=definition_matching.id) - sample_other = _sample_record(definition_id=definition_other.id) - - # Capture IDs before the session closes to avoid DetachedInstanceError - matching_definition_id = str(definition_matching.id) - matching_id = str(sample_matching.id) - other_id = str(sample_other.id) - - with session_factory() as session: - session.add(definition_matching) - session.add(definition_other) - session.add(sample_matching) - session.add(sample_other) - session.commit() - - monkeypatch.setattr(core_sample_views, "get_session", session_factory) - - rc = sample_cmd.list_samples_command( - Namespace(definition_id=matching_definition_id, experiment=None, status=None, limit=20) - ) - - assert rc == 0 - out = capsys.readouterr().out - assert matching_id in out - assert other_id not in out - - def test_sample_list_filters_by_experiment_tag(monkeypatch, session_factory, capsys): """The experiment filter reads the v2 ``SampleRecord.experiment`` tag.""" definition = _definition(name="matching") @@ -243,9 +213,7 @@ def test_sample_list_filters_by_experiment_tag(monkeypatch, session_factory, cap monkeypatch.setattr(core_sample_views, "get_session", session_factory) - rc = sample_cmd.list_samples_command( - Namespace(definition_id=None, experiment="alpha", status=None, limit=20) - ) + rc = sample_cmd.list_samples_command(Namespace(experiment="alpha", status=None, limit=20)) assert rc == 0 out = capsys.readouterr().out