PR 09: Add sample-centered CLI inspection

ergon_cli/ergon_cli/domains/experiments/commands.py

@@ -1,15 +1,15 @@
 from argparse import Namespace
 
 from ergon_cli.domains.experiments.models import (
-    ListByTagCommand,
+    ExperimentSamplesCommand,
+    ExperimentSamplerInvocationsCommand,
     ListExperimentsCommand,
-    ListTagsCommand,
     ShowExperimentCommand,
 )
 from ergon_cli.domains.experiments.service import (
-    list_by_tag,
+    list_experiment_samples,
+    list_experiment_sampler_invocations,
     list_experiments,
-    list_tags,
     show_experiment,
 )
 from ergon_cli.shared import exit_codes
@@ -23,106 +23,130 @@ async def handle_experiment(args: Namespace) -> int:
         return handle_experiment_show(args)
     if args.experiment_action == "list":
         return handle_experiment_list(args)
-    if args.experiment_action == "tags":
-        return handle_experiment_tags(args)
-    if args.experiment_action == "by-tag":
-        return handle_experiment_by_tag(args)
-    print("Usage: ergon experiment {show|list|tags|by-tag}")
+    if args.experiment_action == "samples":
+        return handle_experiment_samples(args)
+    if args.experiment_action == "sampler-invocations":
+        return handle_experiment_sampler_invocations(args)
+    print("Usage: ergon experiment {show|list|samples|sampler-invocations}")
     return exit_codes.RUNTIME_ERROR
 
 
 def handle_experiment_show(args: Namespace) -> int:
     try:
         detail = show_experiment(
             ShowExperimentCommand(
-                definition_id=parse_uuid(args.definition_id, field_name="definition_id")
+                experiment_id=parse_uuid(args.experiment_id, field_name="experiment_id")
             )
         )
     except CliError as exc:
         print(exc.message)
         return exc.exit_code
-    experiment = detail.experiment
     lines = [
-        f"DEFINITION_ID={experiment.definition_id}",
-        f"NAME={experiment.name}",
-        f"BENCHMARK={experiment.benchmark_type}",
-        f"STATUS={experiment.status}",
-        f"SAMPLE_COUNT={experiment.sample_count}",
-        f"RUN_COUNT={experiment.run_count}",
+        f"EXPERIMENT_ID={detail.experiment_id}",
+        f"NAME={detail.name}",
+        f"SAMPLE_COUNT={detail.sample_count}",
+        f"SAMPLER_INVOCATIONS={detail.sampler_invocation_count}",
     ]
-    if experiment.default_model_target is not None:
-        lines.append(f"DEFAULT_MODEL={experiment.default_model_target}")
-    if experiment.default_evaluator_slug is not None:
-        lines.append(f"DEFAULT_EVALUATOR={experiment.default_evaluator_slug}")
-    if detail.sample_selection:
-        lines.append(f"SAMPLE_SELECTION={detail.sample_selection}")
-    if detail.runs:
+    if detail.environments:
+        lines.append("ENVIRONMENTS")
+        lines.extend(
+            "\t".join(
+                [
+                    environment.environment_name,
+                    environment.source_mode,
+                    str(environment.sample_count),
+                    str(environment.selected_count),
+                ]
+            )
+            for environment in detail.environments
+        )
+    if detail.samples:
         lines.append("SAMPLES")
         lines.extend(
             "\t".join(
                 [
-                    str(run.sample_id),
-                    run.instance_key,
-                    run.status,
-                    "" if run.model_target is None else run.model_target,
+                    str(sample.sample_id),
+                    sample.environment_name,
+                    sample.sample_key,
+                    sample.status,
                 ]
             )
-            for run in detail.runs
+            for sample in detail.samples
         )
     print(render_text(lines))
     return exit_codes.OK
 
 
-def handle_experiment_list(args: Namespace) -> int:
-    experiments = list_experiments(ListExperimentsCommand(limit=args.limit))
-    if not experiments:
-        print("No experiments found.")
+def handle_experiment_samples(args: Namespace) -> int:
+    try:
+        result = list_experiment_samples(
+            ExperimentSamplesCommand(
+                experiment_id=parse_uuid(args.experiment_id, field_name="experiment_id")
+            )
+        )
+    except CliError as exc:
+        print(exc.message)
+        return exc.exit_code
+    if not result.samples:
+        print("No samples found.")
         return exit_codes.OK
-    lines = ["DEFINITION_ID\tNAME\tBENCHMARK\tSTATUS\tSAMPLES\tATTEMPTS\tMODEL"]
+    lines = ["SAMPLE_ID\tENVIRONMENT\tKEY\tSTATUS"]
     lines.extend(
         "\t".join(
-            [
-                str(experiment.definition_id),
-                experiment.name,
-                experiment.benchmark_type,
-                experiment.status,
-                str(experiment.sample_count),
-                str(experiment.run_count),
-                "" if experiment.default_model_target is None else experiment.default_model_target,
-            ]
+            [str(sample.sample_id), sample.environment_name, sample.sample_key, sample.status]
         )
-        for experiment in experiments
+        for sample in result.samples
     )
     print(render_text(lines))
     return exit_codes.OK
 
 
-def handle_experiment_tags(args: Namespace) -> int:
-    del args
-    tags = list_tags(ListTagsCommand())
-    if not tags:
-        print("No experiment tags found.")
+def handle_experiment_sampler_invocations(args: Namespace) -> int:
+    try:
+        result = list_experiment_sampler_invocations(
+            ExperimentSamplerInvocationsCommand(
+                experiment_id=parse_uuid(args.experiment_id, field_name="experiment_id")
+            )
+        )
+    except CliError as exc:
+        print(exc.message)
+        return exc.exit_code
+    if not result.invocations:
+        print("No sampler invocations found.")
         return exit_codes.OK
-    print(render_text(tags))
+    lines = ["SAMPLER_INVOCATION_ID\tSAMPLER\tK\tCANDIDATES\tSELECTED"]
+    lines.extend(
+        "\t".join(
+            [
+                str(invocation.sampler_invocation_id),
+                invocation.sampler_name,
+                str(invocation.requested_k),
+                str(invocation.candidate_pool_size),
+                str(invocation.selected_count),
+            ]
+        )
+        for invocation in result.invocations
+    )
+    print(render_text(lines))
     return exit_codes.OK
 
 
-def handle_experiment_by_tag(args: Namespace) -> int:
-    rows = list_by_tag(ListByTagCommand(tag=args.tag))
-    if not rows:
-        print(f"No definitions found for experiment tag {args.tag!r}.")
+def handle_experiment_list(args: Namespace) -> int:
+    result = list_experiments(ListExperimentsCommand(limit=args.limit))
+    if not result.experiments:
+        print("No experiments found.")
         return exit_codes.OK
-    lines = ["DEFINITION_ID\tNAME\tBENCHMARK\tLATEST_RUN_STATUS"]
+    lines = ["EXPERIMENT_ID\tNAME\tSAMPLES\tSAMPLER_INVOCATIONS"]
     lines.extend(
         "\t".join(
             [
-                str(row.definition_id),
-                row.name,
-                row.benchmark_type,
-                "" if row.latest_run_status is None else row.latest_run_status,
+                str(experiment.experiment_id),
+                experiment.name,
+                str(experiment.sample_count),
+                str(experiment.sampler_invocation_count),
             ]
         )
-        for row in rows
+        for experiment in result.experiments
     )
     print(render_text(lines))
     return exit_codes.OK

ergon_cli/ergon_cli/domains/experiments/models.py

@@ -10,52 +10,72 @@ class ListExperimentsCommand(BaseModel):
 
 class ShowExperimentCommand(BaseModel):
     model_config = ConfigDict(frozen=True)
-    definition_id: UUID
+    experiment_id: UUID
 
 
-class ListTagsCommand(BaseModel):
+class ExperimentSamplesCommand(BaseModel):
     model_config = ConfigDict(frozen=True)
+    experiment_id: UUID
 
 
-class ListByTagCommand(BaseModel):
+class ExperimentSamplerInvocationsCommand(BaseModel):
     model_config = ConfigDict(frozen=True)
-    tag: str
+    experiment_id: UUID
 
 
-class ExperimentSummaryView(BaseModel):
+class ExperimentEnvironmentCliState(BaseModel):
     model_config = ConfigDict(frozen=True)
 
-    definition_id: UUID
-    name: str
-    benchmark_type: str
-    status: str
+    environment_id: UUID
+    environment_name: str
+    source_mode: str
     sample_count: int
-    run_count: int
-    default_model_target: str | None = None
-    default_evaluator_slug: str | None = None
+    selected_count: int
 
 
-class ExperimentRunView(BaseModel):
+class ExperimentSampleCliState(BaseModel):
     model_config = ConfigDict(frozen=True)
 
     sample_id: UUID
-    instance_key: str
+    environment_name: str
+    sample_key: str
     status: str
-    model_target: str | None = None
 
 
-class ExperimentDetailView(BaseModel):
+class SamplerInvocationCliState(BaseModel):
     model_config = ConfigDict(frozen=True)
 
-    experiment: ExperimentSummaryView
-    sample_selection: dict
-    runs: tuple[ExperimentRunView, ...]
+    sampler_invocation_id: UUID
+    sampler_name: str
+    requested_k: int
+    candidate_pool_size: int
+    selected_count: int
 
 
-class ExperimentTagDefinitionView(BaseModel):
+class ExperimentCliState(BaseModel):
     model_config = ConfigDict(frozen=True)
 
-    definition_id: UUID
+    experiment_id: UUID
     name: str
-    benchmark_type: str
-    latest_run_status: str | None = None
+    environments: tuple[ExperimentEnvironmentCliState, ...]
+    sample_count: int
+    sampler_invocation_count: int
+    samples: tuple[ExperimentSampleCliState, ...] = ()
+
+
+class ExperimentListCliResult(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    experiments: tuple[ExperimentCliState, ...]
+
+
+class ExperimentSamplesCliResult(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    samples: tuple[ExperimentSampleCliState, ...]
+
+
+class ExperimentSamplerInvocationsCliResult(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    invocations: tuple[SamplerInvocationCliState, ...]

ergon_cli/ergon_cli/domains/experiments/parser.py

@@ -8,11 +8,13 @@ def register_experiment_parser(subparsers: argparse._SubParsersAction) -> None:
     experiment.set_defaults(handler=handle_experiment)
     experiment_sub = experiment.add_subparsers(dest="experiment_action")
     experiment_show = experiment_sub.add_parser("show", help="Show experiment detail")
-    experiment_show.add_argument("definition_id", help="Experiment UUID")
+    experiment_show.add_argument("experiment_id", help="Experiment UUID")
     experiment_list = experiment_sub.add_parser("list", help="List experiments")
     experiment_list.add_argument("--limit", type=int, default=50, help="Number of experiments")
-    experiment_sub.add_parser("tags", help="List distinct experiment tags")
-    experiment_by_tag = experiment_sub.add_parser(
-        "by-tag", help="List definitions for an experiment tag"
+    experiment_samples = experiment_sub.add_parser("samples", help="List samples in an experiment")
+    experiment_samples.add_argument("experiment_id", help="Experiment UUID")
+    experiment_invocations = experiment_sub.add_parser(
+        "sampler-invocations",
+        help="List sampler invocations for an experiment",
     )
-    experiment_by_tag.add_argument("tag", help="Experiment tag")
+    experiment_invocations.add_argument("experiment_id", help="Experiment UUID")