From 941e78aaea41f18104240d5467ab312f18fcefc2 Mon Sep 17 00:00:00 2001
From: gziv <gziv@redhat.com>
Date: Wed, 15 Apr 2026 15:35:52 +0300
Subject: [PATCH 1/7] feat: add eval config generation script and tests

Python script that reads metadata.yaml and generates a Harbor job config
YAML for A/B evaluation. Supports two modes: prebuilt (digest image refs
from build-push) and local-build (Harbor builds from Dockerfiles).

Extracts n_trials, timeout multipliers, and resource overrides from the
submission's ExperimentConfig and SubmissionMetadata schemas.
---
 scripts/generate_eval_config.py    | 221 +++++++++++++++++++++++
 tests/test_generate_eval_config.py | 276 +++++++++++++++++++++++++++++
 2 files changed, 497 insertions(+)
 create mode 100644 scripts/generate_eval_config.py
 create mode 100644 tests/test_generate_eval_config.py

diff --git a/scripts/generate_eval_config.py b/scripts/generate_eval_config.py
new file mode 100644
index 0000000..4ca4893
--- /dev/null
+++ b/scripts/generate_eval_config.py
@@ -0,0 +1,221 @@
+"""Generate a Harbor job config YAML for A/B evaluation.
+
+Reads metadata.yaml from the submission directory and produces a config
+file that can be passed to ``harbor run -c <config.yaml>``.
+
+Usage:
+    python scripts/generate_eval_config.py \
+        --submission-dir /workspace/submissions/my-submission \
+        --treatment-task-dir /workspace/tasks-treatment/my-submission \
+        --control-task-dir /workspace/tasks-control/my-submission \
+        --eval-mode prebuilt \
+        --treatment-image-ref registry/ns/img@sha256:abc \
+        --control-image-ref registry/ns/img@sha256:def \
+        --output /tmp/eval-config.yaml
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from abevalflow.schemas import SubmissionMetadata
+
+logger = logging.getLogger(__name__)
+
+EVAL_MODES = ("prebuilt", "local-build")
+
+# Harbor's default agent/verifier/setup timeouts used as reference baselines
+# for computing timeout multipliers from the absolute seconds in metadata.yaml.
+_HARBOR_DEFAULT_AGENT_TIMEOUT = 600.0
+_HARBOR_DEFAULT_VERIFIER_TIMEOUT = 120.0
+_HARBOR_DEFAULT_SETUP_TIMEOUT = 600.0
+_HARBOR_DEFAULT_BUILD_TIMEOUT = 600.0
+
+
+def load_metadata(submission_dir: Path) -> SubmissionMetadata:
+    """Load and validate metadata.yaml from a submission directory."""
+    meta_path = submission_dir / "metadata.yaml"
+    with meta_path.open() as f:
+        raw = yaml.safe_load(f)
+    return SubmissionMetadata(**raw)
+
+
+def _timeout_multiplier(actual: float, default: float) -> float:
+    """Compute a timeout multiplier relative to Harbor's default.
+
+    Returns 1.0 when actual equals the default, >1.0 for longer timeouts.
+    """
+    if default <= 0:
+        return 1.0
+    return actual / default
+
+
+def build_eval_config(
+    metadata: SubmissionMetadata,
+    treatment_task_dir: str,
+    control_task_dir: str,
+    eval_mode: str,
+    jobs_dir: str = "jobs",
+    treatment_image_ref: str = "",
+    control_image_ref: str = "",
+) -> dict[str, Any]:
+    """Build a dict matching Harbor's JobConfig schema."""
+    treatment_task: dict[str, Any] = {"path": treatment_task_dir}
+    control_task: dict[str, Any] = {"path": control_task_dir}
+
+    if eval_mode == "prebuilt":
+        treatment_task["environment_kwargs"] = {"image_ref": treatment_image_ref}
+        control_task["environment_kwargs"] = {"image_ref": control_image_ref}
+
+    agent_mult = _timeout_multiplier(
+        metadata.agent_timeout_sec, _HARBOR_DEFAULT_AGENT_TIMEOUT
+    )
+    verifier_mult = _timeout_multiplier(
+        metadata.verifier_timeout_sec, _HARBOR_DEFAULT_VERIFIER_TIMEOUT
+    )
+    setup_mult = _timeout_multiplier(
+        metadata.agent_setup_timeout_sec, _HARBOR_DEFAULT_SETUP_TIMEOUT
+    )
+    build_mult = _timeout_multiplier(
+        metadata.build_timeout_sec, _HARBOR_DEFAULT_BUILD_TIMEOUT
+    )
+
+    config: dict[str, Any] = {
+        "job_name": f"{metadata.name}-eval",
+        "jobs_dir": jobs_dir,
+        "n_attempts": metadata.experiment.n_trials,
+        "timeout_multiplier": 1.0,
+        "agent_timeout_multiplier": agent_mult,
+        "verifier_timeout_multiplier": verifier_mult,
+        "agent_setup_timeout_multiplier": setup_mult,
+        "environment_build_timeout_multiplier": build_mult,
+        "n_concurrent_trials": 4,
+        "environment": {
+            "type": "openshift",
+            "delete": True,
+            "override_cpus": metadata.cpus,
+            "override_memory_mb": metadata.memory_mb,
+            "override_storage_mb": metadata.storage_mb,
+        },
+        "agents": [{}],
+        "tasks": [treatment_task, control_task],
+    }
+
+    if eval_mode == "local-build":
+        config["environment"]["force_build"] = True
+
+    return config
+
+
+def generate_eval_config(
+    submission_dir: Path,
+    treatment_task_dir: str,
+    control_task_dir: str,
+    output: Path,
+    eval_mode: str,
+    treatment_image_ref: str = "",
+    control_image_ref: str = "",
+    jobs_dir: str = "jobs",
+) -> dict[str, Any]:
+    """End-to-end: load metadata, build config, write YAML, return the dict."""
+    metadata = load_metadata(submission_dir)
+    config = build_eval_config(
+        metadata=metadata,
+        treatment_task_dir=treatment_task_dir,
+        control_task_dir=control_task_dir,
+        eval_mode=eval_mode,
+        jobs_dir=jobs_dir,
+        treatment_image_ref=treatment_image_ref,
+        control_image_ref=control_image_ref,
+    )
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with output.open("w") as f:
+        yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+    logger.info("Wrote eval config to %s", output)
+    return config
+
+
+def main(argv: list[str] | None = None) -> int:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+    parser = argparse.ArgumentParser(
+        description="Generate a Harbor eval config from submission metadata",
+    )
+    parser.add_argument(
+        "--submission-dir",
+        type=Path,
+        required=True,
+        help="Path to the submission directory containing metadata.yaml",
+    )
+    parser.add_argument(
+        "--treatment-task-dir",
+        required=True,
+        help="Path to the scaffolded treatment task directory",
+    )
+    parser.add_argument(
+        "--control-task-dir",
+        required=True,
+        help="Path to the scaffolded control task directory",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        required=True,
+        help="Path to write the generated config YAML",
+    )
+    parser.add_argument(
+        "--eval-mode",
+        choices=EVAL_MODES,
+        required=True,
+        help="'prebuilt' uses digest image refs; 'local-build' lets Harbor build from Dockerfiles",
+    )
+    parser.add_argument(
+        "--treatment-image-ref",
+        default="",
+        help="Digest-based image ref for treatment variant (required for prebuilt mode)",
+    )
+    parser.add_argument(
+        "--control-image-ref",
+        default="",
+        help="Digest-based image ref for control variant (required for prebuilt mode)",
+    )
+    parser.add_argument(
+        "--jobs-dir",
+        default="jobs",
+        help="Directory where Harbor writes job results (default: jobs)",
+    )
+
+    args = parser.parse_args(argv)
+
+    if args.eval_mode == "prebuilt":
+        if not args.treatment_image_ref or not args.control_image_ref:
+            parser.error(
+                "--treatment-image-ref and --control-image-ref are required "
+                "when --eval-mode is 'prebuilt'"
+            )
+
+    if not args.submission_dir.is_dir():
+        logger.error("Submission directory does not exist: %s", args.submission_dir)
+        return 1
+
+    generate_eval_config(
+        submission_dir=args.submission_dir,
+        treatment_task_dir=args.treatment_task_dir,
+        control_task_dir=args.control_task_dir,
+        output=args.output,
+        eval_mode=args.eval_mode,
+        treatment_image_ref=args.treatment_image_ref,
+        control_image_ref=args.control_image_ref,
+        jobs_dir=args.jobs_dir,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tests/test_generate_eval_config.py b/tests/test_generate_eval_config.py
new file mode 100644
index 0000000..70e9173
--- /dev/null
+++ b/tests/test_generate_eval_config.py
@@ -0,0 +1,276 @@
+"""Tests for scripts/generate_eval_config.py — Harbor eval config generation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+import yaml
+
+from scripts.generate_eval_config import (
+    build_eval_config,
+    generate_eval_config,
+    load_metadata,
+    main,
+)
+
+
+@pytest.fixture()
+def minimal_submission(tmp_path: Path) -> Path:
+    """Submission with only a name — all defaults."""
+    sub = tmp_path / "my-submission"
+    sub.mkdir()
+    (sub / "metadata.yaml").write_text(yaml.dump({"name": "my-submission"}))
+    return sub
+
+
+@pytest.fixture()
+def custom_submission(tmp_path: Path) -> Path:
+    """Submission with custom experiment and resource config."""
+    sub = tmp_path / "custom-eval"
+    sub.mkdir()
+    meta = {
+        "name": "custom-eval",
+        "description": "A custom evaluation",
+        "experiment": {"n_trials": 10, "type": "model"},
+        "agent_timeout_sec": 1200.0,
+        "verifier_timeout_sec": 240.0,
+        "agent_setup_timeout_sec": 300.0,
+        "build_timeout_sec": 900.0,
+        "cpus": 2,
+        "memory_mb": 4096,
+        "storage_mb": 20480,
+    }
+    (sub / "metadata.yaml").write_text(yaml.dump(meta))
+    return sub
+
+
+TREATMENT_DIR = "/workspace/tasks-treatment/my-submission"
+CONTROL_DIR = "/workspace/tasks-control/my-submission"
+TREATMENT_REF = "registry.example.com/ns/my-submission@sha256:aaa111"
+CONTROL_REF = "registry.example.com/ns/my-submission@sha256:bbb222"
+
+
+class TestLoadMetadata:
+    def test_loads_minimal(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        assert meta.name == "my-submission"
+        assert meta.experiment.n_trials == 20
+
+    def test_loads_custom(self, custom_submission: Path):
+        meta = load_metadata(custom_submission)
+        assert meta.name == "custom-eval"
+        assert meta.experiment.n_trials == 10
+        assert meta.cpus == 2
+
+    def test_missing_metadata_raises(self, tmp_path: Path):
+        sub = tmp_path / "empty"
+        sub.mkdir()
+        with pytest.raises(FileNotFoundError):
+            load_metadata(sub)
+
+
+class TestBuildEvalConfigPrebuilt:
+    def test_basic_structure(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR,
+            eval_mode="prebuilt",
+            treatment_image_ref=TREATMENT_REF,
+            control_image_ref=CONTROL_REF,
+        )
+        assert config["job_name"] == "my-submission-eval"
+        assert config["n_attempts"] == 20
+        assert config["environment"]["type"] == "openshift"
+        assert config["environment"]["delete"] is True
+        assert len(config["tasks"]) == 2
+
+    def test_treatment_task_has_image_ref(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR,
+            eval_mode="prebuilt",
+            treatment_image_ref=TREATMENT_REF,
+            control_image_ref=CONTROL_REF,
+        )
+        treatment = config["tasks"][0]
+        assert treatment["path"] == TREATMENT_DIR
+        assert treatment["environment_kwargs"]["image_ref"] == TREATMENT_REF
+
+    def test_control_task_has_image_ref(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR,
+            eval_mode="prebuilt",
+            treatment_image_ref=TREATMENT_REF,
+            control_image_ref=CONTROL_REF,
+        )
+        control = config["tasks"][1]
+        assert control["path"] == CONTROL_DIR
+        assert control["environment_kwargs"]["image_ref"] == CONTROL_REF
+
+    def test_no_force_build(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR,
+            eval_mode="prebuilt",
+            treatment_image_ref=TREATMENT_REF,
+            control_image_ref=CONTROL_REF,
+        )
+        assert "force_build" not in config["environment"]
+
+
+class TestBuildEvalConfigLocalBuild:
+    def test_no_environment_kwargs(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        )
+        assert "environment_kwargs" not in config["tasks"][0]
+        assert "environment_kwargs" not in config["tasks"][1]
+
+    def test_force_build_enabled(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        )
+        assert config["environment"]["force_build"] is True
+
+    def test_task_paths_set(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        )
+        assert config["tasks"][0]["path"] == TREATMENT_DIR
+        assert config["tasks"][1]["path"] == CONTROL_DIR
+
+
+class TestCustomMetadataFields:
+    def test_n_trials_from_metadata(self, custom_submission: Path):
+        meta = load_metadata(custom_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        )
+        assert config["n_attempts"] == 10
+
+    def test_resource_overrides(self, custom_submission: Path):
+        meta = load_metadata(custom_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        )
+        assert config["environment"]["override_cpus"] == 2
+        assert config["environment"]["override_memory_mb"] == 4096
+        assert config["environment"]["override_storage_mb"] == 20480
+
+    def test_timeout_multipliers(self, custom_submission: Path):
+        meta = load_metadata(custom_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        )
+        assert config["agent_timeout_multiplier"] == pytest.approx(2.0)
+        assert config["verifier_timeout_multiplier"] == pytest.approx(2.0)
+        assert config["agent_setup_timeout_multiplier"] == pytest.approx(0.5)
+        assert config["environment_build_timeout_multiplier"] == pytest.approx(1.5)
+
+    def test_default_timeouts_produce_1x_multiplier(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        )
+        assert config["agent_timeout_multiplier"] == pytest.approx(1.0)
+        assert config["verifier_timeout_multiplier"] == pytest.approx(1.0)
+        assert config["agent_setup_timeout_multiplier"] == pytest.approx(1.0)
+        assert config["environment_build_timeout_multiplier"] == pytest.approx(1.0)
+
+    def test_custom_jobs_dir(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        config = build_eval_config(
+            meta, TREATMENT_DIR, CONTROL_DIR,
+            eval_mode="local-build", jobs_dir="/workspace/results",
+        )
+        assert config["jobs_dir"] == "/workspace/results"
+
+
+class TestGenerateEvalConfig:
+    def test_writes_yaml_file(self, minimal_submission: Path, tmp_path: Path):
+        output = tmp_path / "config.yaml"
+        config = generate_eval_config(
+            submission_dir=minimal_submission,
+            treatment_task_dir=TREATMENT_DIR,
+            control_task_dir=CONTROL_DIR,
+            output=output,
+            eval_mode="prebuilt",
+            treatment_image_ref=TREATMENT_REF,
+            control_image_ref=CONTROL_REF,
+        )
+        assert output.is_file()
+        loaded = yaml.safe_load(output.read_text())
+        assert loaded["job_name"] == config["job_name"]
+        assert loaded["n_attempts"] == config["n_attempts"]
+        assert len(loaded["tasks"]) == 2
+
+    def test_creates_parent_dirs(self, minimal_submission: Path, tmp_path: Path):
+        output = tmp_path / "nested" / "dir" / "config.yaml"
+        generate_eval_config(
+            submission_dir=minimal_submission,
+            treatment_task_dir=TREATMENT_DIR,
+            control_task_dir=CONTROL_DIR,
+            output=output,
+            eval_mode="local-build",
+        )
+        assert output.is_file()
+
+
+class TestMainCLI:
+    def test_prebuilt_mode(self, minimal_submission: Path, tmp_path: Path):
+        output = tmp_path / "out.yaml"
+        rc = main([
+            "--submission-dir", str(minimal_submission),
+            "--treatment-task-dir", TREATMENT_DIR,
+            "--control-task-dir", CONTROL_DIR,
+            "--output", str(output),
+            "--eval-mode", "prebuilt",
+            "--treatment-image-ref", TREATMENT_REF,
+            "--control-image-ref", CONTROL_REF,
+        ])
+        assert rc == 0
+        loaded = yaml.safe_load(output.read_text())
+        assert loaded["tasks"][0]["environment_kwargs"]["image_ref"] == TREATMENT_REF
+
+    def test_local_build_mode(self, minimal_submission: Path, tmp_path: Path):
+        output = tmp_path / "out.yaml"
+        rc = main([
+            "--submission-dir", str(minimal_submission),
+            "--treatment-task-dir", TREATMENT_DIR,
+            "--control-task-dir", CONTROL_DIR,
+            "--output", str(output),
+            "--eval-mode", "local-build",
+        ])
+        assert rc == 0
+        loaded = yaml.safe_load(output.read_text())
+        assert "environment_kwargs" not in loaded["tasks"][0]
+
+    def test_prebuilt_missing_refs_exits_error(
+        self, minimal_submission: Path, tmp_path: Path,
+    ):
+        output = tmp_path / "out.yaml"
+        with pytest.raises(SystemExit) as exc_info:
+            main([
+                "--submission-dir", str(minimal_submission),
+                "--treatment-task-dir", TREATMENT_DIR,
+                "--control-task-dir", CONTROL_DIR,
+                "--output", str(output),
+                "--eval-mode", "prebuilt",
+            ])
+        assert exc_info.value.code == 2
+
+    def test_nonexistent_submission_dir(self, tmp_path: Path):
+        output = tmp_path / "out.yaml"
+        rc = main([
+            "--submission-dir", str(tmp_path / "no-such-dir"),
+            "--treatment-task-dir", TREATMENT_DIR,
+            "--control-task-dir", CONTROL_DIR,
+            "--output", str(output),
+            "--eval-mode", "local-build",
+        ])
+        assert rc == 1

From 88d2ecad98c0f514ca1bf4343ee5be21985536c0 Mon Sep 17 00:00:00 2001
From: gziv <gziv@redhat.com>
Date: Wed, 15 Apr 2026 15:35:58 +0300
Subject: [PATCH 2/7] feat: add harbor-eval Tekton task

Three-step Tekton task that installs Harbor from the fork, generates a
job config from submission metadata, and runs treatment + control trials.
Parses Harbor result.json files to emit pass rates as Tekton results.
---
 pipeline/tasks/harbor-eval.yaml | 237 ++++++++++++++++++++++++++++++++
 1 file changed, 237 insertions(+)
 create mode 100644 pipeline/tasks/harbor-eval.yaml

diff --git a/pipeline/tasks/harbor-eval.yaml b/pipeline/tasks/harbor-eval.yaml
new file mode 100644
index 0000000..af9114e
--- /dev/null
+++ b/pipeline/tasks/harbor-eval.yaml
@@ -0,0 +1,237 @@
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: harbor-eval
+  namespace: ab-eval-flow
+spec:
+  description: >-
+    Runs A/B evaluation using Harbor on OpenShift. Installs Harbor from the
+    fork repo, generates a job config from submission metadata, and executes
+    treatment + control trials. Supports two modes: prebuilt (uses digest-based
+    image refs from the build-push task) and local-build (Harbor builds from
+    each task's Dockerfile).
+  params:
+    - name: submission-dir
+      type: string
+      description: Path to the submission directory within the workspace (e.g. my-submission)
+    - name: submission-name
+      type: string
+      description: Validated submission name (from validate-submission results)
+    - name: treatment-task-dir
+      type: string
+      description: Path to the scaffolded treatment task directory
+    - name: control-task-dir
+      type: string
+      description: Path to the scaffolded control task directory
+    - name: treatment-image-ref
+      type: string
+      default: ""
+      description: Digest-based image ref for treatment variant (required for prebuilt mode)
+    - name: control-image-ref
+      type: string
+      default: ""
+      description: Digest-based image ref for control variant (required for prebuilt mode)
+    - name: eval-mode
+      type: string
+      default: "prebuilt"
+      description: >-
+        Evaluation mode: 'prebuilt' uses digest image refs from build-push;
+        'local-build' lets Harbor build from each task's Dockerfile.
+    - name: harbor-fork-url
+      type: string
+      default: "https://github.com/GuyZivRH/skills_eval_corrections.git"
+      description: Harbor fork repository URL
+    - name: harbor-fork-revision
+      type: string
+      default: "main"
+      description: Branch or SHA of the Harbor fork to install
+    - name: pipeline-repo-url
+      type: string
+      default: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git"
+      description: URL of the ABEvalFlow pipeline repository
+    - name: pipeline-repo-revision
+      type: string
+      default: "main"
+      description: Branch or SHA of the pipeline repo to use for scripts
+  workspaces:
+    - name: source
+      description: Workspace containing the cloned submissions and scaffolded task directories
+  results:
+    - name: treatment-pass-rate
+      description: Treatment variant pass rate as a decimal string (e.g. "0.85")
+    - name: control-pass-rate
+      description: Control variant pass rate as a decimal string (e.g. "0.60")
+    - name: jobs-dir
+      description: Path to the Harbor jobs output directory
+  steps:
+    - name: install-harbor
+      image: registry.access.redhat.com/ubi9/python-311:latest
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        echo "Installing Harbor from $(params.harbor-fork-url) @ $(params.harbor-fork-revision)"
+        INSTALL_START=$(date +%s)
+
+        pip install --quiet --no-cache-dir \
+          "git+$(params.harbor-fork-url)@$(params.harbor-fork-revision)"
+
+        INSTALL_END=$(date +%s)
+        echo "Harbor installed in $((INSTALL_END - INSTALL_START))s"
+        harbor --version
+
+    - name: generate-config
+      image: registry.access.redhat.com/ubi9/python-311:latest
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        PIPELINE_DIR="$(workspaces.source.path)/_pipeline"
+        if [ -d "$PIPELINE_DIR" ]; then
+          echo "Pipeline repo already cloned, skipping"
+        else
+          git clone --depth 1 --branch "$(params.pipeline-repo-revision)" \
+            "$(params.pipeline-repo-url)" "$PIPELINE_DIR"
+        fi
+
+        SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)"
+        JOBS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)"
+
+        cd "$PIPELINE_DIR"
+        pip install --quiet --no-cache-dir pydantic pyyaml
+
+        ARGS=(
+          --submission-dir "$SUBMISSION_PATH"
+          --treatment-task-dir "$(params.treatment-task-dir)"
+          --control-task-dir "$(params.control-task-dir)"
+          --output /tmp/eval-config.yaml
+          --eval-mode "$(params.eval-mode)"
+          --jobs-dir "$JOBS_DIR"
+        )
+
+        if [ "$(params.eval-mode)" = "prebuilt" ]; then
+          ARGS+=(
+            --treatment-image-ref "$(params.treatment-image-ref)"
+            --control-image-ref "$(params.control-image-ref)"
+          )
+        fi
+
+        python scripts/generate_eval_config.py "${ARGS[@]}"
+
+        echo "--- Generated config ---"
+        cat /tmp/eval-config.yaml
+
+    - name: run-evaluation
+      image: registry.access.redhat.com/ubi9/python-311:latest
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+
+        JOBS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)"
+
+        echo "Starting Harbor evaluation (mode=$(params.eval-mode))"
+        EVAL_START=$(date +%s)
+
+        harbor run -c /tmp/eval-config.yaml --quiet -y
+
+        EVAL_END=$(date +%s)
+        echo "Evaluation completed in $((EVAL_END - EVAL_START))s"
+
+        echo -n "$JOBS_DIR" > "$(results.jobs-dir.path)"
+
+        # Parse results: count pass/fail per variant from result.json files.
+        # Harbor writes one directory per trial; each contains result.json
+        # with verifier_result.reward (1.0 = pass, 0.0 = fail).
+        python - "$JOBS_DIR" "$(params.submission-name)" \
+          "$(results.treatment-pass-rate.path)" \
+          "$(results.control-pass-rate.path)" <<'PARSE_SCRIPT'
+        import json
+        import sys
+        from pathlib import Path
+
+        jobs_dir = Path(sys.argv[1])
+        submission_name = sys.argv[2]
+        treatment_result_path = sys.argv[3]
+        control_result_path = sys.argv[4]
+
+        def compute_pass_rate(job_dir: Path) -> str:
+            """Scan trial directories and compute pass rate from result.json files."""
+            if not job_dir.is_dir():
+                return "0.0"
+            total = 0
+            passed = 0
+            for trial_dir in sorted(job_dir.iterdir()):
+                if not trial_dir.is_dir():
+                    continue
+                result_file = trial_dir / "result.json"
+                if not result_file.is_file():
+                    continue
+                try:
+                    data = json.loads(result_file.read_text())
+                    vr = data.get("verifier_result", {})
+                    reward = vr.get("reward")
+                    total += 1
+                    if reward is not None and float(reward) > 0.0:
+                        passed += 1
+                except (json.JSONDecodeError, ValueError, TypeError):
+                    total += 1
+            if total == 0:
+                return "0.0"
+            return f"{passed / total:.4f}"
+
+        # Find the job directory — Harbor creates a timestamped subdir.
+        # Scan for trial dirs that match each variant by task directory name.
+        # Trial dir names follow the pattern: <task-name>__<shortuuid>
+        treatment_total = 0
+        treatment_passed = 0
+        control_total = 0
+        control_passed = 0
+
+        # Harbor writes to jobs_dir/<job_name>/ — find the most recent job
+        job_subdirs = sorted(
+            [d for d in jobs_dir.iterdir() if d.is_dir()],
+            key=lambda d: d.stat().st_mtime,
+            reverse=True,
+        ) if jobs_dir.is_dir() else []
+
+        if job_subdirs:
+            job_dir = job_subdirs[0]
+            for trial_dir in sorted(job_dir.iterdir()):
+                if not trial_dir.is_dir():
+                    continue
+                result_file = trial_dir / "result.json"
+                if not result_file.is_file():
+                    continue
+                try:
+                    data = json.loads(result_file.read_text())
+                    vr = data.get("verifier_result", {})
+                    reward = vr.get("reward")
+                    is_pass = reward is not None and float(reward) > 0.0
+                except (json.JSONDecodeError, ValueError, TypeError):
+                    is_pass = False
+
+                trial_name = trial_dir.name
+                if "tasks-treatment" in trial_name or trial_name.startswith(submission_name):
+                    treatment_total += 1
+                    if is_pass:
+                        treatment_passed += 1
+                elif "tasks-control" in trial_name:
+                    control_total += 1
+                    if is_pass:
+                        control_passed += 1
+                else:
+                    treatment_total += 1
+                    if is_pass:
+                        treatment_passed += 1
+
+        t_rate = f"{treatment_passed / treatment_total:.4f}" if treatment_total > 0 else "0.0"
+        c_rate = f"{control_passed / control_total:.4f}" if control_total > 0 else "0.0"
+
+        with open(treatment_result_path, "w") as f:
+            f.write(t_rate)
+        with open(control_result_path, "w") as f:
+            f.write(c_rate)
+
+        print(f"Treatment: {treatment_passed}/{treatment_total} = {t_rate}")
+        print(f"Control:   {control_passed}/{control_total} = {c_rate}")
+        PARSE_SCRIPT

From 1d17b95155b2529f976841b604b3802d4b8745be Mon Sep 17 00:00:00 2001
From: gziv <gziv@redhat.com>
Date: Wed, 15 Apr 2026 15:36:03 +0300
Subject: [PATCH 3/7] feat: add RBAC for trial Pod lifecycle

Add pipeline-trial-manager Role and RoleBinding granting the pipeline
ServiceAccount permissions for trial Pod lifecycle management: pods,
pods/exec, pods/log (create/get/list/watch/delete), events (get/list),
and secrets (get for LLM credentials).
---
 config/rbac.yaml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/config/rbac.yaml b/config/rbac.yaml
index e500c63..633ca92 100644
--- a/config/rbac.yaml
+++ b/config/rbac.yaml
@@ -11,3 +11,33 @@ subjects:
   - kind: ServiceAccount
     name: pipeline
     namespace: ab-eval-flow
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pipeline-trial-manager
+  namespace: ab-eval-flow
+rules:
+  - apiGroups: [""]
+    resources: [pods, pods/exec, pods/log]
+    verbs: [create, get, list, watch, delete]
+  - apiGroups: [""]
+    resources: [events]
+    verbs: [get, list]
+  - apiGroups: [""]
+    resources: [secrets]
+    verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pipeline-trial-manager
+  namespace: ab-eval-flow
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pipeline-trial-manager
+subjects:
+  - kind: ServiceAccount
+    name: pipeline
+    namespace: ab-eval-flow

From 45f18a3ceeeafcde7d7360d8250bda46d0c60cb8 Mon Sep 17 00:00:00 2001
From: gziv <gziv@redhat.com>
Date: Wed, 15 Apr 2026 15:36:11 +0300
Subject: [PATCH 4/7] docs: add harbor fork requirements and update
 implementation plan

Document the per-task environment_kwargs change needed in the Harbor
fork's TaskConfig for per-variant image refs in a single sweep config.
Also document WS3A doc alignment items (naming, RBAC, security context).

Update Phase 4 checkboxes: mark fork integration (PR #1), Tekton task,
eval config script, RBAC, and dual eval mode support as done.
---
 Docs/harbor_fork_requirements.md | 101 +++++++++++++++++++++++++++++++
 Docs/implementation_plan.md      |  37 ++++++-----
 2 files changed, 122 insertions(+), 16 deletions(-)
 create mode 100644 Docs/harbor_fork_requirements.md

diff --git a/Docs/harbor_fork_requirements.md b/Docs/harbor_fork_requirements.md
new file mode 100644
index 0000000..573188c
--- /dev/null
+++ b/Docs/harbor_fork_requirements.md
@@ -0,0 +1,101 @@
+# Harbor Fork — Required Changes for ABEvalFlow Integration
+
+> **Target repo:** [RHEcosystemAppEng/skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections)
+> **Open PR:** [#1 — feat: add OpenShift environment backend](https://github.com/RHEcosystemAppEng/skills_eval_corrections/pull/1)
+> **ABEvalFlow branch:** `APPENG-4906/harbor-eval-task`
+
+---
+
+## 1. Per-Task `environment_kwargs` Support
+
+### Problem
+
+ABEvalFlow evaluates two variants (treatment and control) with different pre-built
+images in a single Harbor job. The current `--ek image_ref=<ref>` mechanism sets the
+image ref globally on `EnvironmentConfig.kwargs`, so all tasks in a job share the
+same image. This prevents using a single sweep config for both variants.
+
+### Proposed Change
+
+Add an optional `environment_kwargs` field to `TaskConfig` in
+`src/harbor/models/trial/config.py`:
+
+```python
+class TaskConfig(BaseModel):
+    path: Path
+    git_url: str | None = None
+    git_commit_id: str | None = None
+    overwrite: bool = False
+    download_dir: Path | None = None
+    source: str | None = None
+    environment_kwargs: dict[str, Any] = Field(default_factory=dict)  # <-- new
+```
+
+### Merge Logic
+
+When the `Job` creates `TrialConfig` instances from `JobConfig`, it should merge
+per-task kwargs into the trial's environment config:
+
+```python
+# In the trial creation loop (conceptual):
+trial_env_config = job_config.environment.model_copy(deep=True)
+trial_env_config.kwargs = {**trial_env_config.kwargs, **task.environment_kwargs}
+```
+
+Task-level kwargs override global kwargs for the same key (task wins).
+
+### Result
+
+A single sweep config can specify different `image_ref` values per variant:
+
+```yaml
+job_name: my-submission-eval
+jobs_dir: eval-results
+n_attempts: 20
+environment:
+  type: openshift
+  delete: true
+agents:
+  - name: claude-code
+    model_name: claude-sonnet-4-5
+tasks:
+  - path: /workspace/tasks-treatment/my-submission
+    environment_kwargs:
+      image_ref: "registry/ns/my-submission@sha256:abc..."
+  - path: /workspace/tasks-control/my-submission
+    environment_kwargs:
+      image_ref: "registry/ns/my-submission@sha256:def..."
+```
+
+### Testing
+
+- Unit test: verify that `TrialConfig` created from a `TaskConfig` with
+  `environment_kwargs` has them merged into `environment.kwargs`.
+- Unit test: verify task-level kwargs override global kwargs.
+- Unit test: verify `TaskConfig` without `environment_kwargs` behaves identically
+  to current behavior (backwards compatible).
+
+---
+
+## 2. Handoff Doc Alignment (WS3A)
+
+The existing handoff doc (`Docs/harbor_openshift_backend.md`) has diverged from
+the actual implementation in PR #1. These items should be updated:
+
+| Section | Current (outdated) | Correct |
+|---------|-------------------|---------|
+| File path | `openshift_environment.py` | `openshift.py` |
+| Build modes | `_build_and_push_image` is no-op only | Supports pre-built (`--ek image_ref=`) AND local podman build |
+| Pod security | `readOnlyRootFilesystem: true` | Intentionally unset — many agent workloads need writes; `HOME=/tmp` is injected instead |
+| RBAC table | ConfigMaps, Secrets, PVCs, ImageStreams | Only Pods + exec + Secrets used in practice |
+| Naming | "skilled / unskilled" | "treatment / control" |
+| Trial count | "20 skilled + 20 unskilled" | "20 treatment + 20 control" |
+| Tekton params | `skilled-image-ref` / `unskilled-image-ref` | `treatment-image-ref` / `control-image-ref` |
+| Definition of Done | "20 skilled + 20 unskilled" | "20 treatment + 20 control" |
+
+### Additional Notes
+
+- The OpenShift backend supports a `cpu_request` kwarg (`--ek cpu_request=<val>`)
+  for clusters with tight resource constraints — not documented in the handoff doc.
+- The `--ek registry=<url>` kwarg enables local podman build+push to a specified
+  registry — also undocumented.
diff --git a/Docs/implementation_plan.md b/Docs/implementation_plan.md
index 06c8d7a..d6e7909 100644
--- a/Docs/implementation_plan.md
+++ b/Docs/implementation_plan.md
@@ -300,10 +300,11 @@ Additional requirements:
 
 ### 4.2 Harbor Fork Integration
 
-- [ ] Add `openshift_environment.py` in `src/harbor/environments/`.
-- [ ] Add `OPENSHIFT` to `EnvironmentType` enum.
-- [ ] Register the backend so `harbor run --env openshift` selects it.
+- [x] Add `openshift.py` in `src/harbor/environments/` (PR #1 in fork).
+- [x] Add `OPENSHIFT` to `EnvironmentType` enum (PR #1 in fork).
+- [x] Register the backend so `harbor run --env openshift` selects it (PR #1 in fork).
 - [ ] Pin a fork SHA in the pipeline image; review upstream quarterly for drift.
+- [ ] Add per-task `environment_kwargs` to `TaskConfig` (see `Docs/harbor_fork_requirements.md`).
 
 ### 4.3 Testing Strategy
 
@@ -312,24 +313,28 @@ Additional requirements:
 
 ### 4.4 Trial Execution Configuration
 
-- The `harbor-eval` Tekton task accepts `treatment-image-ref` and `control-image-ref` as **params** wired from Phase 3 results.
-- N = configurable attempts per variant (default 20, treatment + control = 40 total sessions).
-- Configure resource requests/limits per trial Pod.
-- LLM endpoint configured via environment variable — backend is agnostic to whether it points to LiteLLM, a direct API, or a self-hosted model.
-- Trial Pod timeout: configurable, with a global evaluation timeout.
+- [x] `harbor-eval` Tekton task accepts `treatment-image-ref` and `control-image-ref` as params wired from Phase 3 results.
+- [x] N = configurable attempts per variant (default 20, treatment + control = 40 total sessions) via `n-trials` from `metadata.yaml`.
+- [x] Resource requests/limits per trial Pod (from `metadata.yaml`: `cpus`, `memory_mb`, `storage_mb`).
+- [ ] LLM endpoint configured via environment variable — backend is agnostic to whether it points to LiteLLM, a direct API, or a self-hosted model.
+- [x] Trial Pod timeout: configurable via timeout multipliers derived from `metadata.yaml`.
+- [x] Eval config generation script (`scripts/generate_eval_config.py`) reads metadata and produces Harbor job config YAML.
+- [x] Supports two modes: `prebuilt` (digest image refs) and `local-build` (Harbor builds from Dockerfiles).
 
 ### 4.5 RBAC Requirements
 
+- [x] `pipeline-trial-manager` Role + RoleBinding added to `config/rbac.yaml`.
+
 The pipeline ServiceAccount needs (prefer named Secrets for least-privilege where policy requires):
 
-| Resource | Verbs | Purpose |
-|---|---|---|
-| Pods | create, get, list, watch, delete | Trial Pod lifecycle (watch for efficient state tracking) |
-| ConfigMaps | get, list | Trial configuration |
-| Secrets | get | LLM credentials injection via `envFrom` |
-| Events | get, list | Diagnosing hung/failed trial Pods |
-| PVCs | get, list, create | Pipeline workspaces and artifacts |
-| ImageStreams (OpenShift) | get, list | Registry access — validate against target cluster's API version |
+| Resource | Verbs | Purpose | Status |
+|---|---|---|---|
+| Pods, Pods/exec, Pods/log | create, get, list, watch, delete | Trial Pod lifecycle | Done |
+| Secrets | get | LLM credentials injection via `envFrom` | Done |
+| Events | get, list | Diagnosing hung/failed trial Pods | Done |
+| ConfigMaps | get, list | Trial configuration | Deferred — not used by current backend |
+| PVCs | get, list, create | Pipeline workspaces and artifacts | Deferred — handled by Tekton |
+| ImageStreams (OpenShift) | get, list | Registry access | Deferred — not used by current backend |
 
 ### 4.6 Definition of Done
 

From 3cc9d941a1a4461fcc8e93a4e0bdd97566043d73 Mon Sep 17 00:00:00 2001
From: gziv <gziv@redhat.com>
Date: Wed, 15 Apr 2026 15:51:13 +0300
Subject: [PATCH 5/7] fix: address review findings for harbor-eval task

- Collapse three Tekton steps into one to fix step isolation (harbor
  binary and config file were not shared across containers)
- Split evaluation into separate Harbor jobs per variant with dedicated
  results directories, eliminating fragile trial classification heuristic
- Add validation guard in build_variant_config for prebuilt mode
- Remove dead compute_pass_rate function from inline parser
- Pin ubi9/python-311 image to :9.6
- Split RBAC verbs per subresource (pods/exec -> create, pods/log -> get)
- Remove --quiet flag to preserve diagnostic output
- Add comments for n_concurrent_trials and agents defaults
- Fix fork URL to GuyZivRH/skills_eval_corrections
- Downgrade per-task environment_kwargs to nice-to-have in docs
---
 Docs/harbor_fork_requirements.md   |  24 +--
 config/rbac.yaml                   |   8 +-
 pipeline/tasks/harbor-eval.yaml    | 150 ++++++-------------
 scripts/generate_eval_config.py    | 160 ++++++++++++--------
 tests/test_generate_eval_config.py | 229 +++++++++++++++++------------
 5 files changed, 305 insertions(+), 266 deletions(-)

diff --git a/Docs/harbor_fork_requirements.md b/Docs/harbor_fork_requirements.md
index 573188c..32c46ea 100644
--- a/Docs/harbor_fork_requirements.md
+++ b/Docs/harbor_fork_requirements.md
@@ -1,19 +1,23 @@
 # Harbor Fork — Required Changes for ABEvalFlow Integration
 
-> **Target repo:** [RHEcosystemAppEng/skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections)
+> **Target repo:** [GuyZivRH/skills_eval_corrections](https://github.com/GuyZivRH/skills_eval_corrections)
 > **Open PR:** [#1 — feat: add OpenShift environment backend](https://github.com/RHEcosystemAppEng/skills_eval_corrections/pull/1)
 > **ABEvalFlow branch:** `APPENG-4906/harbor-eval-task`
 
 ---
 
-## 1. Per-Task `environment_kwargs` Support
+## 1. Per-Task `environment_kwargs` Support (nice-to-have)
 
-### Problem
+### Context
 
-ABEvalFlow evaluates two variants (treatment and control) with different pre-built
-images in a single Harbor job. The current `--ek image_ref=<ref>` mechanism sets the
-image ref globally on `EnvironmentConfig.kwargs`, so all tasks in a job share the
-same image. This prevents using a single sweep config for both variants.
+ABEvalFlow currently runs each variant (treatment/control) as a **separate Harbor
+job**, each with its own config file and `jobs_dir`. This avoids the need for
+per-task environment kwargs — the global `environment.kwargs.image_ref` works
+because each job has a single task.
+
+However, if we later want to run both variants in a **single Harbor job** (e.g.,
+for sweep-based workflows or simplified config), per-task `environment_kwargs`
+would be needed.
 
 ### Proposed Change
 
@@ -33,7 +37,7 @@ class TaskConfig(BaseModel):
 
 ### Merge Logic
 
-When the `Job` creates `TrialConfig` instances from `JobConfig`, it should merge
+When the `Job` creates `TrialConfig` instances from `JobConfig`, merge
 per-task kwargs into the trial's environment config:
 
 ```python
@@ -44,9 +48,7 @@ trial_env_config.kwargs = {**trial_env_config.kwargs, **task.environment_kwargs}
 
 Task-level kwargs override global kwargs for the same key (task wins).
 
-### Result
-
-A single sweep config can specify different `image_ref` values per variant:
+### Example
 
 ```yaml
 job_name: my-submission-eval
diff --git a/config/rbac.yaml b/config/rbac.yaml
index 633ca92..0b313e8 100644
--- a/config/rbac.yaml
+++ b/config/rbac.yaml
@@ -19,8 +19,14 @@ metadata:
   namespace: ab-eval-flow
 rules:
   - apiGroups: [""]
-    resources: [pods, pods/exec, pods/log]
+    resources: [pods]
     verbs: [create, get, list, watch, delete]
+  - apiGroups: [""]
+    resources: [pods/exec]
+    verbs: [create]
+  - apiGroups: [""]
+    resources: [pods/log]
+    verbs: [get]
   - apiGroups: [""]
     resources: [events]
     verbs: [get, list]
diff --git a/pipeline/tasks/harbor-eval.yaml b/pipeline/tasks/harbor-eval.yaml
index af9114e..c061f97 100644
--- a/pipeline/tasks/harbor-eval.yaml
+++ b/pipeline/tasks/harbor-eval.yaml
@@ -6,10 +6,11 @@ metadata:
 spec:
   description: >-
     Runs A/B evaluation using Harbor on OpenShift. Installs Harbor from the
-    fork repo, generates a job config from submission metadata, and executes
-    treatment + control trials. Supports two modes: prebuilt (uses digest-based
-    image refs from the build-push task) and local-build (Harbor builds from
-    each task's Dockerfile).
+    fork repo, generates per-variant job configs from submission metadata,
+    and executes treatment and control as separate Harbor jobs. Each variant
+    gets its own results directory for clean trial attribution. Supports two
+    modes: prebuilt (uses digest-based image refs from the build-push task)
+    and local-build (Harbor builds from each task's Dockerfile).
   params:
     - name: submission-dir
       type: string
@@ -39,11 +40,14 @@ spec:
         'local-build' lets Harbor build from each task's Dockerfile.
     - name: harbor-fork-url
       type: string
-      default: "https://github.com/GuyZivRH/skills_eval_corrections.git"
+      default: "https://github.com/RHEcosystemAppEng/skills_eval_corrections.git"
       description: Harbor fork repository URL
     - name: harbor-fork-revision
       type: string
       default: "main"
+      # TODO: pin to a specific SHA once the fork stabilises — "main" means
+      # each TaskRun pulls whatever is current, risking breakage from
+      # upstream changes.
       description: Branch or SHA of the Harbor fork to install
     - name: pipeline-repo-url
       type: string
@@ -61,31 +65,25 @@ spec:
       description: Treatment variant pass rate as a decimal string (e.g. "0.85")
     - name: control-pass-rate
       description: Control variant pass rate as a decimal string (e.g. "0.60")
-    - name: jobs-dir
-      description: Path to the Harbor jobs output directory
+    - name: results-dir
+      description: Path to the base results directory containing treatment/ and control/ subdirs
   steps:
-    - name: install-harbor
-      image: registry.access.redhat.com/ubi9/python-311:latest
+    - name: run-evaluation
+      image: registry.access.redhat.com/ubi9/python-311:9.6
       script: |
         #!/usr/bin/env bash
         set -euo pipefail
 
-        echo "Installing Harbor from $(params.harbor-fork-url) @ $(params.harbor-fork-revision)"
+        # --- 1. Install Harbor from fork ---
+        echo "=== Installing Harbor from $(params.harbor-fork-url) @ $(params.harbor-fork-revision) ==="
         INSTALL_START=$(date +%s)
-
         pip install --quiet --no-cache-dir \
           "git+$(params.harbor-fork-url)@$(params.harbor-fork-revision)"
-
         INSTALL_END=$(date +%s)
         echo "Harbor installed in $((INSTALL_END - INSTALL_START))s"
         harbor --version
 
-    - name: generate-config
-      image: registry.access.redhat.com/ubi9/python-311:latest
-      script: |
-        #!/usr/bin/env bash
-        set -euo pipefail
-
+        # --- 2. Clone pipeline repo and generate per-variant configs ---
         PIPELINE_DIR="$(workspaces.source.path)/_pipeline"
         if [ -d "$PIPELINE_DIR" ]; then
           echo "Pipeline repo already cloned, skipping"
@@ -95,7 +93,8 @@ spec:
         fi
 
         SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)"
-        JOBS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)"
+        RESULTS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)"
+        CONFIG_DIR="$(workspaces.source.path)/_eval-configs"
 
         cd "$PIPELINE_DIR"
         pip install --quiet --no-cache-dir pydantic pyyaml
@@ -104,9 +103,9 @@ spec:
           --submission-dir "$SUBMISSION_PATH"
           --treatment-task-dir "$(params.treatment-task-dir)"
           --control-task-dir "$(params.control-task-dir)"
-          --output /tmp/eval-config.yaml
+          --output-dir "$CONFIG_DIR"
           --eval-mode "$(params.eval-mode)"
-          --jobs-dir "$JOBS_DIR"
+          --results-base-dir "$RESULTS_DIR"
         )
 
         if [ "$(params.eval-mode)" = "prebuilt" ]; then
@@ -118,54 +117,46 @@ spec:
 
         python scripts/generate_eval_config.py "${ARGS[@]}"
 
-        echo "--- Generated config ---"
-        cat /tmp/eval-config.yaml
+        echo "=== Treatment config ==="
+        cat "$CONFIG_DIR/treatment-config.yaml"
+        echo "=== Control config ==="
+        cat "$CONFIG_DIR/control-config.yaml"
 
-    - name: run-evaluation
-      image: registry.access.redhat.com/ubi9/python-311:latest
-      script: |
-        #!/usr/bin/env bash
-        set -euo pipefail
-
-        JOBS_DIR="$(workspaces.source.path)/eval-results/$(params.submission-name)"
-
-        echo "Starting Harbor evaluation (mode=$(params.eval-mode))"
+        # --- 3. Run treatment evaluation ---
+        echo "=== Running treatment evaluation ==="
         EVAL_START=$(date +%s)
+        harbor run -c "$CONFIG_DIR/treatment-config.yaml" -y
+        EVAL_MID=$(date +%s)
+        echo "Treatment completed in $((EVAL_MID - EVAL_START))s"
 
-        harbor run -c /tmp/eval-config.yaml --quiet -y
-
+        # --- 4. Run control evaluation ---
+        echo "=== Running control evaluation ==="
+        harbor run -c "$CONFIG_DIR/control-config.yaml" -y
         EVAL_END=$(date +%s)
-        echo "Evaluation completed in $((EVAL_END - EVAL_START))s"
+        echo "Control completed in $((EVAL_END - EVAL_MID))s"
+        echo "Total evaluation time: $((EVAL_END - EVAL_START))s"
 
-        echo -n "$JOBS_DIR" > "$(results.jobs-dir.path)"
+        # --- 5. Parse results ---
+        echo -n "$RESULTS_DIR" > "$(results.results-dir.path)"
 
-        # Parse results: count pass/fail per variant from result.json files.
-        # Harbor writes one directory per trial; each contains result.json
-        # with verifier_result.reward (1.0 = pass, 0.0 = fail).
-        python - "$JOBS_DIR" "$(params.submission-name)" \
+        python - "$RESULTS_DIR" \
           "$(results.treatment-pass-rate.path)" \
           "$(results.control-pass-rate.path)" <<'PARSE_SCRIPT'
         import json
         import sys
         from pathlib import Path
 
-        jobs_dir = Path(sys.argv[1])
-        submission_name = sys.argv[2]
-        treatment_result_path = sys.argv[3]
-        control_result_path = sys.argv[4]
+        results_dir = Path(sys.argv[1])
+        treatment_result_path = sys.argv[2]
+        control_result_path = sys.argv[3]
 
-        def compute_pass_rate(job_dir: Path) -> str:
-            """Scan trial directories and compute pass rate from result.json files."""
-            if not job_dir.is_dir():
+        def compute_pass_rate(variant_dir: Path) -> str:
+            """Scan all trial result.json files under a variant's jobs directory."""
+            if not variant_dir.is_dir():
                 return "0.0"
             total = 0
             passed = 0
-            for trial_dir in sorted(job_dir.iterdir()):
-                if not trial_dir.is_dir():
-                    continue
-                result_file = trial_dir / "result.json"
-                if not result_file.is_file():
-                    continue
+            for result_file in sorted(variant_dir.rglob("result.json")):
                 try:
                     data = json.loads(result_file.read_text())
                     vr = data.get("verifier_result", {})
@@ -179,59 +170,14 @@ spec:
                 return "0.0"
             return f"{passed / total:.4f}"
 
-        # Find the job directory — Harbor creates a timestamped subdir.
-        # Scan for trial dirs that match each variant by task directory name.
-        # Trial dir names follow the pattern: <task-name>__<shortuuid>
-        treatment_total = 0
-        treatment_passed = 0
-        control_total = 0
-        control_passed = 0
-
-        # Harbor writes to jobs_dir/<job_name>/ — find the most recent job
-        job_subdirs = sorted(
-            [d for d in jobs_dir.iterdir() if d.is_dir()],
-            key=lambda d: d.stat().st_mtime,
-            reverse=True,
-        ) if jobs_dir.is_dir() else []
-
-        if job_subdirs:
-            job_dir = job_subdirs[0]
-            for trial_dir in sorted(job_dir.iterdir()):
-                if not trial_dir.is_dir():
-                    continue
-                result_file = trial_dir / "result.json"
-                if not result_file.is_file():
-                    continue
-                try:
-                    data = json.loads(result_file.read_text())
-                    vr = data.get("verifier_result", {})
-                    reward = vr.get("reward")
-                    is_pass = reward is not None and float(reward) > 0.0
-                except (json.JSONDecodeError, ValueError, TypeError):
-                    is_pass = False
-
-                trial_name = trial_dir.name
-                if "tasks-treatment" in trial_name or trial_name.startswith(submission_name):
-                    treatment_total += 1
-                    if is_pass:
-                        treatment_passed += 1
-                elif "tasks-control" in trial_name:
-                    control_total += 1
-                    if is_pass:
-                        control_passed += 1
-                else:
-                    treatment_total += 1
-                    if is_pass:
-                        treatment_passed += 1
-
-        t_rate = f"{treatment_passed / treatment_total:.4f}" if treatment_total > 0 else "0.0"
-        c_rate = f"{control_passed / control_total:.4f}" if control_total > 0 else "0.0"
+        t_rate = compute_pass_rate(results_dir / "treatment")
+        c_rate = compute_pass_rate(results_dir / "control")
 
         with open(treatment_result_path, "w") as f:
             f.write(t_rate)
         with open(control_result_path, "w") as f:
             f.write(c_rate)
 
-        print(f"Treatment: {treatment_passed}/{treatment_total} = {t_rate}")
-        print(f"Control:   {control_passed}/{control_total} = {c_rate}")
+        print(f"Treatment pass rate: {t_rate}")
+        print(f"Control pass rate:   {c_rate}")
         PARSE_SCRIPT
diff --git a/scripts/generate_eval_config.py b/scripts/generate_eval_config.py
index 4ca4893..72bf39c 100644
--- a/scripts/generate_eval_config.py
+++ b/scripts/generate_eval_config.py
@@ -1,17 +1,30 @@
-"""Generate a Harbor job config YAML for A/B evaluation.
+"""Generate per-variant Harbor job configs for A/B evaluation.
 
-Reads metadata.yaml from the submission directory and produces a config
-file that can be passed to ``harbor run -c <config.yaml>``.
+Reads metadata.yaml from the submission directory and produces two config
+files (one per variant) that can be passed to ``harbor run -c <config.yaml>``.
+
+Each variant runs as a separate Harbor job with its own jobs directory,
+producing a clean result layout::
+
+    <results-base-dir>/
+        treatment/
+            <job-name>/
+                <trial-1>__<uuid>/result.json
+                ...
+        control/
+            <job-name>/
+                <trial-1>__<uuid>/result.json
+                ...
 
 Usage:
-    python scripts/generate_eval_config.py \
-        --submission-dir /workspace/submissions/my-submission \
-        --treatment-task-dir /workspace/tasks-treatment/my-submission \
-        --control-task-dir /workspace/tasks-control/my-submission \
-        --eval-mode prebuilt \
-        --treatment-image-ref registry/ns/img@sha256:abc \
-        --control-image-ref registry/ns/img@sha256:def \
-        --output /tmp/eval-config.yaml
+    python scripts/generate_eval_config.py \\
+        --submission-dir /workspace/submissions/my-submission \\
+        --treatment-task-dir /workspace/tasks-treatment/my-submission \\
+        --control-task-dir /workspace/tasks-control/my-submission \\
+        --eval-mode prebuilt \\
+        --treatment-image-ref registry/ns/img@sha256:abc \\
+        --control-image-ref registry/ns/img@sha256:def \\
+        --output-dir /workspace/eval-configs
 """
 
 from __future__ import annotations
@@ -29,6 +42,7 @@
 logger = logging.getLogger(__name__)
 
 EVAL_MODES = ("prebuilt", "local-build")
+VARIANTS = ("treatment", "control")
 
 # Harbor's default agent/verifier/setup timeouts used as reference baselines
 # for computing timeout multipliers from the absolute seconds in metadata.yaml.
@@ -56,22 +70,38 @@ def _timeout_multiplier(actual: float, default: float) -> float:
     return actual / default
 
 
-def build_eval_config(
+def build_variant_config(
     metadata: SubmissionMetadata,
-    treatment_task_dir: str,
-    control_task_dir: str,
+    variant: str,
+    task_dir: str,
     eval_mode: str,
-    jobs_dir: str = "jobs",
-    treatment_image_ref: str = "",
-    control_image_ref: str = "",
+    jobs_dir: str,
+    image_ref: str = "",
 ) -> dict[str, Any]:
-    """Build a dict matching Harbor's JobConfig schema."""
-    treatment_task: dict[str, Any] = {"path": treatment_task_dir}
-    control_task: dict[str, Any] = {"path": control_task_dir}
+    """Build a Harbor JobConfig dict for a single variant.
+
+    Each variant gets its own job so results land in a variant-specific
+    directory and trial classification is unambiguous.
+    """
+    if eval_mode == "prebuilt" and not image_ref:
+        raise ValueError(
+            f"image_ref is required for variant '{variant}' in prebuilt mode"
+        )
+
+    task: dict[str, Any] = {"path": task_dir}
+
+    env_block: dict[str, Any] = {
+        "type": "openshift",
+        "delete": True,
+        "override_cpus": metadata.cpus,
+        "override_memory_mb": metadata.memory_mb,
+        "override_storage_mb": metadata.storage_mb,
+    }
 
     if eval_mode == "prebuilt":
-        treatment_task["environment_kwargs"] = {"image_ref": treatment_image_ref}
-        control_task["environment_kwargs"] = {"image_ref": control_image_ref}
+        env_block["kwargs"] = {"image_ref": image_ref}
+    else:
+        env_block["force_build"] = True
 
     agent_mult = _timeout_multiplier(
         metadata.agent_timeout_sec, _HARBOR_DEFAULT_AGENT_TIMEOUT
@@ -86,8 +116,12 @@ def build_eval_config(
         metadata.build_timeout_sec, _HARBOR_DEFAULT_BUILD_TIMEOUT
     )
 
+    # n_concurrent_trials=4 is Harbor's default; kept explicit so operators
+    # can tune it per-cluster via a future CLI param or metadata field.
+    # agents=[{}] inherits Harbor's default agent (oracle agent); the
+    # pipeline will wire agent_name/model via params in a future iteration.
     config: dict[str, Any] = {
-        "job_name": f"{metadata.name}-eval",
+        "job_name": f"{metadata.name}-{variant}",
         "jobs_dir": jobs_dir,
         "n_attempts": metadata.experiment.n_trials,
         "timeout_multiplier": 1.0,
@@ -96,56 +130,58 @@ def build_eval_config(
         "agent_setup_timeout_multiplier": setup_mult,
         "environment_build_timeout_multiplier": build_mult,
         "n_concurrent_trials": 4,
-        "environment": {
-            "type": "openshift",
-            "delete": True,
-            "override_cpus": metadata.cpus,
-            "override_memory_mb": metadata.memory_mb,
-            "override_storage_mb": metadata.storage_mb,
-        },
+        "environment": env_block,
         "agents": [{}],
-        "tasks": [treatment_task, control_task],
+        "tasks": [task],
     }
 
-    if eval_mode == "local-build":
-        config["environment"]["force_build"] = True
-
     return config
 
 
-def generate_eval_config(
+def generate_eval_configs(
     submission_dir: Path,
     treatment_task_dir: str,
     control_task_dir: str,
-    output: Path,
+    output_dir: Path,
     eval_mode: str,
+    results_base_dir: str,
     treatment_image_ref: str = "",
     control_image_ref: str = "",
-    jobs_dir: str = "jobs",
-) -> dict[str, Any]:
-    """End-to-end: load metadata, build config, write YAML, return the dict."""
+) -> dict[str, dict[str, Any]]:
+    """Generate per-variant Harbor configs, write YAML files, return both."""
     metadata = load_metadata(submission_dir)
-    config = build_eval_config(
-        metadata=metadata,
-        treatment_task_dir=treatment_task_dir,
-        control_task_dir=control_task_dir,
-        eval_mode=eval_mode,
-        jobs_dir=jobs_dir,
-        treatment_image_ref=treatment_image_ref,
-        control_image_ref=control_image_ref,
-    )
-    output.parent.mkdir(parents=True, exist_ok=True)
-    with output.open("w") as f:
-        yaml.dump(config, f, default_flow_style=False, sort_keys=False)
-    logger.info("Wrote eval config to %s", output)
-    return config
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    variant_args = {
+        "treatment": (treatment_task_dir, treatment_image_ref),
+        "control": (control_task_dir, control_image_ref),
+    }
+
+    configs: dict[str, dict[str, Any]] = {}
+    for variant, (task_dir, img_ref) in variant_args.items():
+        jobs_dir = f"{results_base_dir}/{variant}"
+        config = build_variant_config(
+            metadata=metadata,
+            variant=variant,
+            task_dir=task_dir,
+            eval_mode=eval_mode,
+            jobs_dir=jobs_dir,
+            image_ref=img_ref,
+        )
+        out_path = output_dir / f"{variant}-config.yaml"
+        with out_path.open("w") as f:
+            yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+        logger.info("Wrote %s config to %s", variant, out_path)
+        configs[variant] = config
+
+    return configs
 
 
 def main(argv: list[str] | None = None) -> int:
     logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
 
     parser = argparse.ArgumentParser(
-        description="Generate a Harbor eval config from submission metadata",
+        description="Generate per-variant Harbor eval configs from submission metadata",
     )
     parser.add_argument(
         "--submission-dir",
@@ -164,10 +200,10 @@ def main(argv: list[str] | None = None) -> int:
         help="Path to the scaffolded control task directory",
     )
     parser.add_argument(
-        "--output",
+        "--output-dir",
         type=Path,
         required=True,
-        help="Path to write the generated config YAML",
+        help="Directory to write the per-variant config YAML files",
     )
     parser.add_argument(
         "--eval-mode",
@@ -186,9 +222,9 @@ def main(argv: list[str] | None = None) -> int:
         help="Digest-based image ref for control variant (required for prebuilt mode)",
     )
     parser.add_argument(
-        "--jobs-dir",
-        default="jobs",
-        help="Directory where Harbor writes job results (default: jobs)",
+        "--results-base-dir",
+        default="eval-results",
+        help="Base directory for Harbor job results (default: eval-results)",
     )
 
     args = parser.parse_args(argv)
@@ -204,15 +240,15 @@ def main(argv: list[str] | None = None) -> int:
         logger.error("Submission directory does not exist: %s", args.submission_dir)
         return 1
 
-    generate_eval_config(
+    generate_eval_configs(
         submission_dir=args.submission_dir,
         treatment_task_dir=args.treatment_task_dir,
         control_task_dir=args.control_task_dir,
-        output=args.output,
+        output_dir=args.output_dir,
         eval_mode=args.eval_mode,
+        results_base_dir=args.results_base_dir,
         treatment_image_ref=args.treatment_image_ref,
         control_image_ref=args.control_image_ref,
-        jobs_dir=args.jobs_dir,
     )
     return 0
 
diff --git a/tests/test_generate_eval_config.py b/tests/test_generate_eval_config.py
index 70e9173..556a548 100644
--- a/tests/test_generate_eval_config.py
+++ b/tests/test_generate_eval_config.py
@@ -1,4 +1,4 @@
-"""Tests for scripts/generate_eval_config.py — Harbor eval config generation."""
+"""Tests for scripts/generate_eval_config.py — per-variant Harbor eval config generation."""
 
 from __future__ import annotations
 
@@ -8,8 +8,8 @@
 import yaml
 
 from scripts.generate_eval_config import (
-    build_eval_config,
-    generate_eval_config,
+    build_variant_config,
+    generate_eval_configs,
     load_metadata,
     main,
 )
@@ -70,93 +70,86 @@ def test_missing_metadata_raises(self, tmp_path: Path):
             load_metadata(sub)
 
 
-class TestBuildEvalConfigPrebuilt:
+class TestBuildVariantConfigPrebuilt:
     def test_basic_structure(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR,
-            eval_mode="prebuilt",
-            treatment_image_ref=TREATMENT_REF,
-            control_image_ref=CONTROL_REF,
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "prebuilt",
+            jobs_dir="results/treatment", image_ref=TREATMENT_REF,
         )
-        assert config["job_name"] == "my-submission-eval"
+        assert config["job_name"] == "my-submission-treatment"
         assert config["n_attempts"] == 20
         assert config["environment"]["type"] == "openshift"
         assert config["environment"]["delete"] is True
-        assert len(config["tasks"]) == 2
+        assert len(config["tasks"]) == 1
+        assert config["tasks"][0]["path"] == TREATMENT_DIR
 
-    def test_treatment_task_has_image_ref(self, minimal_submission: Path):
+    def test_image_ref_in_env_kwargs(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR,
-            eval_mode="prebuilt",
-            treatment_image_ref=TREATMENT_REF,
-            control_image_ref=CONTROL_REF,
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "prebuilt",
+            jobs_dir="results/treatment", image_ref=TREATMENT_REF,
         )
-        treatment = config["tasks"][0]
-        assert treatment["path"] == TREATMENT_DIR
-        assert treatment["environment_kwargs"]["image_ref"] == TREATMENT_REF
+        assert config["environment"]["kwargs"]["image_ref"] == TREATMENT_REF
 
-    def test_control_task_has_image_ref(self, minimal_submission: Path):
+    def test_control_variant_naming(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR,
-            eval_mode="prebuilt",
-            treatment_image_ref=TREATMENT_REF,
-            control_image_ref=CONTROL_REF,
+        config = build_variant_config(
+            meta, "control", CONTROL_DIR, "prebuilt",
+            jobs_dir="results/control", image_ref=CONTROL_REF,
         )
-        control = config["tasks"][1]
-        assert control["path"] == CONTROL_DIR
-        assert control["environment_kwargs"]["image_ref"] == CONTROL_REF
+        assert config["job_name"] == "my-submission-control"
+        assert config["tasks"][0]["path"] == CONTROL_DIR
 
     def test_no_force_build(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR,
-            eval_mode="prebuilt",
-            treatment_image_ref=TREATMENT_REF,
-            control_image_ref=CONTROL_REF,
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "prebuilt",
+            jobs_dir="results/treatment", image_ref=TREATMENT_REF,
         )
         assert "force_build" not in config["environment"]
 
+    def test_missing_image_ref_raises(self, minimal_submission: Path):
+        meta = load_metadata(minimal_submission)
+        with pytest.raises(ValueError, match="image_ref is required"):
+            build_variant_config(
+                meta, "treatment", TREATMENT_DIR, "prebuilt",
+                jobs_dir="results/treatment", image_ref="",
+            )
+
 
-class TestBuildEvalConfigLocalBuild:
-    def test_no_environment_kwargs(self, minimal_submission: Path):
+class TestBuildVariantConfigLocalBuild:
+    def test_no_env_kwargs(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "local-build",
+            jobs_dir="results/treatment",
         )
-        assert "environment_kwargs" not in config["tasks"][0]
-        assert "environment_kwargs" not in config["tasks"][1]
+        assert "kwargs" not in config["environment"]
 
     def test_force_build_enabled(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "local-build",
+            jobs_dir="results/treatment",
         )
         assert config["environment"]["force_build"] is True
 
-    def test_task_paths_set(self, minimal_submission: Path):
-        meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
-        )
-        assert config["tasks"][0]["path"] == TREATMENT_DIR
-        assert config["tasks"][1]["path"] == CONTROL_DIR
-
 
 class TestCustomMetadataFields:
     def test_n_trials_from_metadata(self, custom_submission: Path):
         meta = load_metadata(custom_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "local-build",
+            jobs_dir="results/treatment",
         )
         assert config["n_attempts"] == 10
 
     def test_resource_overrides(self, custom_submission: Path):
         meta = load_metadata(custom_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "local-build",
+            jobs_dir="results/treatment",
         )
         assert config["environment"]["override_cpus"] == 2
         assert config["environment"]["override_memory_mb"] == 4096
@@ -164,8 +157,9 @@ def test_resource_overrides(self, custom_submission: Path):
 
     def test_timeout_multipliers(self, custom_submission: Path):
         meta = load_metadata(custom_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "local-build",
+            jobs_dir="results/treatment",
         )
         assert config["agent_timeout_multiplier"] == pytest.approx(2.0)
         assert config["verifier_timeout_multiplier"] == pytest.approx(2.0)
@@ -174,8 +168,9 @@ def test_timeout_multipliers(self, custom_submission: Path):
 
     def test_default_timeouts_produce_1x_multiplier(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR, eval_mode="local-build",
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "local-build",
+            jobs_dir="results/treatment",
         )
         assert config["agent_timeout_multiplier"] == pytest.approx(1.0)
         assert config["verifier_timeout_multiplier"] == pytest.approx(1.0)
@@ -184,93 +179,147 @@ def test_default_timeouts_produce_1x_multiplier(self, minimal_submission: Path):
 
     def test_custom_jobs_dir(self, minimal_submission: Path):
         meta = load_metadata(minimal_submission)
-        config = build_eval_config(
-            meta, TREATMENT_DIR, CONTROL_DIR,
-            eval_mode="local-build", jobs_dir="/workspace/results",
+        config = build_variant_config(
+            meta, "treatment", TREATMENT_DIR, "local-build",
+            jobs_dir="/workspace/results/treatment",
+        )
+        assert config["jobs_dir"] == "/workspace/results/treatment"
+
+
+class TestGenerateEvalConfigs:
+    def test_writes_two_yaml_files(self, minimal_submission: Path, tmp_path: Path):
+        out_dir = tmp_path / "configs"
+        configs = generate_eval_configs(
+            submission_dir=minimal_submission,
+            treatment_task_dir=TREATMENT_DIR,
+            control_task_dir=CONTROL_DIR,
+            output_dir=out_dir,
+            eval_mode="prebuilt",
+            results_base_dir="eval-results",
+            treatment_image_ref=TREATMENT_REF,
+            control_image_ref=CONTROL_REF,
         )
-        assert config["jobs_dir"] == "/workspace/results"
+        assert (out_dir / "treatment-config.yaml").is_file()
+        assert (out_dir / "control-config.yaml").is_file()
+        assert "treatment" in configs
+        assert "control" in configs
 
+    def test_variant_jobs_dirs_are_separate(
+        self, minimal_submission: Path, tmp_path: Path,
+    ):
+        out_dir = tmp_path / "configs"
+        configs = generate_eval_configs(
+            submission_dir=minimal_submission,
+            treatment_task_dir=TREATMENT_DIR,
+            control_task_dir=CONTROL_DIR,
+            output_dir=out_dir,
+            eval_mode="prebuilt",
+            results_base_dir="eval-results",
+            treatment_image_ref=TREATMENT_REF,
+            control_image_ref=CONTROL_REF,
+        )
+        assert configs["treatment"]["jobs_dir"] == "eval-results/treatment"
+        assert configs["control"]["jobs_dir"] == "eval-results/control"
 
-class TestGenerateEvalConfig:
-    def test_writes_yaml_file(self, minimal_submission: Path, tmp_path: Path):
-        output = tmp_path / "config.yaml"
-        config = generate_eval_config(
+    def test_each_config_has_single_task(
+        self, minimal_submission: Path, tmp_path: Path,
+    ):
+        out_dir = tmp_path / "configs"
+        configs = generate_eval_configs(
+            submission_dir=minimal_submission,
+            treatment_task_dir=TREATMENT_DIR,
+            control_task_dir=CONTROL_DIR,
+            output_dir=out_dir,
+            eval_mode="local-build",
+            results_base_dir="eval-results",
+        )
+        assert len(configs["treatment"]["tasks"]) == 1
+        assert len(configs["control"]["tasks"]) == 1
+        assert configs["treatment"]["tasks"][0]["path"] == TREATMENT_DIR
+        assert configs["control"]["tasks"][0]["path"] == CONTROL_DIR
+
+    def test_yaml_roundtrips(self, minimal_submission: Path, tmp_path: Path):
+        out_dir = tmp_path / "configs"
+        configs = generate_eval_configs(
             submission_dir=minimal_submission,
             treatment_task_dir=TREATMENT_DIR,
             control_task_dir=CONTROL_DIR,
-            output=output,
+            output_dir=out_dir,
             eval_mode="prebuilt",
+            results_base_dir="eval-results",
             treatment_image_ref=TREATMENT_REF,
             control_image_ref=CONTROL_REF,
         )
-        assert output.is_file()
-        loaded = yaml.safe_load(output.read_text())
-        assert loaded["job_name"] == config["job_name"]
-        assert loaded["n_attempts"] == config["n_attempts"]
-        assert len(loaded["tasks"]) == 2
-
-    def test_creates_parent_dirs(self, minimal_submission: Path, tmp_path: Path):
-        output = tmp_path / "nested" / "dir" / "config.yaml"
-        generate_eval_config(
+        for variant in ("treatment", "control"):
+            loaded = yaml.safe_load(
+                (out_dir / f"{variant}-config.yaml").read_text()
+            )
+            assert loaded["job_name"] == configs[variant]["job_name"]
+            assert loaded["n_attempts"] == configs[variant]["n_attempts"]
+
+    def test_creates_output_dir(self, minimal_submission: Path, tmp_path: Path):
+        out_dir = tmp_path / "nested" / "dir" / "configs"
+        generate_eval_configs(
             submission_dir=minimal_submission,
             treatment_task_dir=TREATMENT_DIR,
             control_task_dir=CONTROL_DIR,
-            output=output,
+            output_dir=out_dir,
             eval_mode="local-build",
+            results_base_dir="eval-results",
         )
-        assert output.is_file()
+        assert (out_dir / "treatment-config.yaml").is_file()
 
 
 class TestMainCLI:
     def test_prebuilt_mode(self, minimal_submission: Path, tmp_path: Path):
-        output = tmp_path / "out.yaml"
+        out_dir = tmp_path / "out"
         rc = main([
             "--submission-dir", str(minimal_submission),
             "--treatment-task-dir", TREATMENT_DIR,
             "--control-task-dir", CONTROL_DIR,
-            "--output", str(output),
+            "--output-dir", str(out_dir),
             "--eval-mode", "prebuilt",
             "--treatment-image-ref", TREATMENT_REF,
             "--control-image-ref", CONTROL_REF,
         ])
         assert rc == 0
-        loaded = yaml.safe_load(output.read_text())
-        assert loaded["tasks"][0]["environment_kwargs"]["image_ref"] == TREATMENT_REF
+        t_config = yaml.safe_load((out_dir / "treatment-config.yaml").read_text())
+        assert t_config["environment"]["kwargs"]["image_ref"] == TREATMENT_REF
 
     def test_local_build_mode(self, minimal_submission: Path, tmp_path: Path):
-        output = tmp_path / "out.yaml"
+        out_dir = tmp_path / "out"
         rc = main([
             "--submission-dir", str(minimal_submission),
             "--treatment-task-dir", TREATMENT_DIR,
             "--control-task-dir", CONTROL_DIR,
-            "--output", str(output),
+            "--output-dir", str(out_dir),
             "--eval-mode", "local-build",
         ])
         assert rc == 0
-        loaded = yaml.safe_load(output.read_text())
-        assert "environment_kwargs" not in loaded["tasks"][0]
+        t_config = yaml.safe_load((out_dir / "treatment-config.yaml").read_text())
+        assert "kwargs" not in t_config["environment"]
 
     def test_prebuilt_missing_refs_exits_error(
         self, minimal_submission: Path, tmp_path: Path,
     ):
-        output = tmp_path / "out.yaml"
+        out_dir = tmp_path / "out"
         with pytest.raises(SystemExit) as exc_info:
             main([
                 "--submission-dir", str(minimal_submission),
                 "--treatment-task-dir", TREATMENT_DIR,
                 "--control-task-dir", CONTROL_DIR,
-                "--output", str(output),
+                "--output-dir", str(out_dir),
                 "--eval-mode", "prebuilt",
             ])
         assert exc_info.value.code == 2
 
     def test_nonexistent_submission_dir(self, tmp_path: Path):
-        output = tmp_path / "out.yaml"
+        out_dir = tmp_path / "out"
         rc = main([
             "--submission-dir", str(tmp_path / "no-such-dir"),
             "--treatment-task-dir", TREATMENT_DIR,
             "--control-task-dir", CONTROL_DIR,
-            "--output", str(output),
+            "--output-dir", str(out_dir),
             "--eval-mode", "local-build",
         ])
         assert rc == 1

From df200257def82bd61e5046dce2c95e28bfca6712 Mon Sep 17 00:00:00 2001
From: gziv <gziv@redhat.com>
Date: Wed, 15 Apr 2026 16:00:54 +0300
Subject: [PATCH 6/7] fix: unify fork URL and expand fork requirements doc

- Use VARIANTS constant in generate_eval_configs loop
- Unify Harbor fork URL to RHEcosystemAppEng/skills_eval_corrections
  across all docs (README, implementation_plan, harbor_openshift_backend,
  harbor_fork_requirements)
- Expand harbor_fork_requirements.md with ABEvalFlow-side implementation
  details: per-variant config structure, result directory layout, metadata
  field mapping, eval modes, and Tekton results emitted
- Document three blocking fork requirements (OpenShift env, config-based
  kwargs passthrough, result.json format) with current status
---
 Docs/harbor_fork_requirements.md | 186 ++++++++++++++++++++++---------
 Docs/harbor_openshift_backend.md |   2 +-
 Docs/implementation_plan.md      |   6 +-
 README.md                        |   2 +-
 scripts/generate_eval_config.py  |   9 +-
 5 files changed, 146 insertions(+), 59 deletions(-)

diff --git a/Docs/harbor_fork_requirements.md b/Docs/harbor_fork_requirements.md
index 32c46ea..0d34d4e 100644
--- a/Docs/harbor_fork_requirements.md
+++ b/Docs/harbor_fork_requirements.md
@@ -1,65 +1,159 @@
-# Harbor Fork — Required Changes for ABEvalFlow Integration
+# Harbor Fork — Integration Requirements for ABEvalFlow
 
-> **Target repo:** [GuyZivRH/skills_eval_corrections](https://github.com/GuyZivRH/skills_eval_corrections)
+> **Target repo:** [RHEcosystemAppEng/skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections)
 > **Open PR:** [#1 — feat: add OpenShift environment backend](https://github.com/RHEcosystemAppEng/skills_eval_corrections/pull/1)
 > **ABEvalFlow branch:** `APPENG-4906/harbor-eval-task`
 
 ---
 
-## 1. Per-Task `environment_kwargs` Support (nice-to-have)
+## ABEvalFlow-Side Implementation (current state)
 
-### Context
+### How ABEvalFlow invokes Harbor
 
-ABEvalFlow currently runs each variant (treatment/control) as a **separate Harbor
-job**, each with its own config file and `jobs_dir`. This avoids the need for
-per-task environment kwargs — the global `environment.kwargs.image_ref` works
-because each job has a single task.
+The `harbor-eval` Tekton task (`pipeline/tasks/harbor-eval.yaml`) runs a single
+step that:
 
-However, if we later want to run both variants in a **single Harbor job** (e.g.,
-for sweep-based workflows or simplified config), per-task `environment_kwargs`
-would be needed.
+1. Installs Harbor from the fork via `pip install git+<fork-url>@<revision>`
+2. Generates **two separate Harbor job configs** (one per variant) using
+   `scripts/generate_eval_config.py`
+3. Runs `harbor run -c treatment-config.yaml` followed by
+   `harbor run -c control-config.yaml`
+4. Parses `result.json` files from each variant's results directory to compute
+   pass rates
 
-### Proposed Change
+### Per-variant config structure
 
-Add an optional `environment_kwargs` field to `TaskConfig` in
-`src/harbor/models/trial/config.py`:
+Each config is a standard Harbor `JobConfig` YAML with a **single task** and
+the image ref set via global `environment.kwargs.image_ref`:
 
-```python
-class TaskConfig(BaseModel):
-    path: Path
-    git_url: str | None = None
-    git_commit_id: str | None = None
-    overwrite: bool = False
-    download_dir: Path | None = None
-    source: str | None = None
-    environment_kwargs: dict[str, Any] = Field(default_factory=dict)  # <-- new
+```yaml
+# treatment-config.yaml
+job_name: my-submission-treatment
+jobs_dir: /workspace/eval-results/my-submission/treatment
+n_attempts: 20
+environment:
+  type: openshift
+  delete: true
+  kwargs:
+    image_ref: "registry/ns/my-submission@sha256:abc..."
+  override_cpus: 1
+  override_memory_mb: 2048
+  override_storage_mb: 10240
+agents:
+  - {}
+tasks:
+  - path: /workspace/tasks-treatment/my-submission
 ```
 
-### Merge Logic
+Control config is identical but with the control image ref and task path.
 
-When the `Job` creates `TrialConfig` instances from `JobConfig`, merge
-per-task kwargs into the trial's environment config:
+### Result directory layout
 
-```python
-# In the trial creation loop (conceptual):
-trial_env_config = job_config.environment.model_copy(deep=True)
-trial_env_config.kwargs = {**trial_env_config.kwargs, **task.environment_kwargs}
 ```
+eval-results/<submission-name>/
+    treatment/
+        <job-name>/
+            <task-name>__<uuid>/result.json
+            <task-name>__<uuid>/result.json
+            ...  (N trials)
+    control/
+        <job-name>/
+            <task-name>__<uuid>/result.json
+            ...  (N trials)
+```
+
+### What ABEvalFlow reads from metadata.yaml
+
+The config generator extracts these fields from `SubmissionMetadata`:
+
+| Field | Maps to | Default |
+|-------|---------|---------|
+| `experiment.n_trials` | `n_attempts` | 20 |
+| `agent_timeout_sec` | `agent_timeout_multiplier` (ratio vs 600s) | 1.0x |
+| `verifier_timeout_sec` | `verifier_timeout_multiplier` (ratio vs 120s) | 1.0x |
+| `agent_setup_timeout_sec` | `agent_setup_timeout_multiplier` (ratio vs 600s) | 1.0x |
+| `build_timeout_sec` | `environment_build_timeout_multiplier` (ratio vs 600s) | 1.0x |
+| `cpus` | `environment.override_cpus` | 1 |
+| `memory_mb` | `environment.override_memory_mb` | 2048 |
+| `storage_mb` | `environment.override_storage_mb` | 10240 |
+
+### Eval modes
+
+| Mode | `--ek image_ref` | Image source | When to use |
+|------|------------------|--------------|-------------|
+| `prebuilt` | Set to digest ref from build-push task | Tekton builds with Buildah, pushes to internal registry | Default pipeline flow |
+| `local-build` | Not set; `force_build: true` | Harbor builds from `environment/Dockerfile` in each task dir | Local dev, or when skipping the build-push step |
+
+### Tekton results emitted
+
+| Result | Description |
+|--------|-------------|
+| `treatment-pass-rate` | Decimal string (e.g. `"0.8500"`) |
+| `control-pass-rate` | Decimal string (e.g. `"0.6000"`) |
+| `results-dir` | Absolute path to the results base directory |
+
+---
+
+## What the Harbor fork must support
+
+### Required (blocking)
+
+**1. `harbor run -c <config.yaml>` with OpenShift environment**
 
-Task-level kwargs override global kwargs for the same key (task wins).
+The fork's `OpenShiftEnvironment` must handle the full trial lifecycle when
+invoked with `--env openshift` (or `environment.type: openshift` in config):
 
-### Example
+- Accept `image_ref` via `environment.kwargs` — verify the image is pullable,
+  skip building
+- Create trial Pods with the pre-built image
+- Execute agent + verifier inside the Pod via `exec`
+- Upload/download files via tar-over-exec
+- Write `result.json` with `verifier_result.reward` (float: 1.0 = pass, 0.0 = fail)
+- Clean up Pods after each trial (`delete: true`)
+
+**Status:** Implemented in PR #1. Needs merge.
+
+**2. `environment.kwargs` passthrough in config-based invocation**
+
+When `harbor run -c config.yaml` is used, the `environment.kwargs` dict from
+the config must be passed to the environment's `__init__`. This is how
+`image_ref` reaches `OpenShiftEnvironment`.
+
+**Status:** Verify this works in `harbor jobs start` with YAML config
+(vs CLI `--ek` flags). The CLI path (`--ek image_ref=...`) is tested;
+the config path should behave identically but needs confirmation.
+
+**3. `result.json` output format**
+
+ABEvalFlow's pass-rate parser expects:
+
+```json
+{
+  "verifier_result": {
+    "reward": 1.0
+  }
+}
+```
+
+Where `reward > 0.0` means pass. This is Harbor's standard format — no change
+needed, but any deviation would break the parser.
+
+### Nice-to-have (not blocking)
+
+**4. Per-task `environment_kwargs` support**
+
+ABEvalFlow currently runs each variant as a separate Harbor job to work around
+the global `environment.kwargs`. If we later want to run both variants in a
+single job (e.g., for sweep-based workflows), per-task `environment_kwargs`
+would be needed.
+
+Proposed change: add `environment_kwargs: dict[str, Any]` to `TaskConfig` in
+`src/harbor/models/trial/config.py`. When `Job` creates `TrialConfig` instances,
+merge per-task kwargs into the trial's `EnvironmentConfig.kwargs` (task-level
+overrides global).
 
 ```yaml
-job_name: my-submission-eval
-jobs_dir: eval-results
-n_attempts: 20
-environment:
-  type: openshift
-  delete: true
-agents:
-  - name: claude-code
-    model_name: claude-sonnet-4-5
+# Single-job example (requires this fork change):
 tasks:
   - path: /workspace/tasks-treatment/my-submission
     environment_kwargs:
@@ -69,17 +163,9 @@ tasks:
       image_ref: "registry/ns/my-submission@sha256:def..."
 ```
 
-### Testing
-
-- Unit test: verify that `TrialConfig` created from a `TaskConfig` with
-  `environment_kwargs` has them merged into `environment.kwargs`.
-- Unit test: verify task-level kwargs override global kwargs.
-- Unit test: verify `TaskConfig` without `environment_kwargs` behaves identically
-  to current behavior (backwards compatible).
-
 ---
 
-## 2. Handoff Doc Alignment (WS3A)
+## Handoff Doc Alignment (WS3A)
 
 The existing handoff doc (`Docs/harbor_openshift_backend.md`) has diverged from
 the actual implementation in PR #1. These items should be updated:
diff --git a/Docs/harbor_openshift_backend.md b/Docs/harbor_openshift_backend.md
index ccbf190..96b08af 100644
--- a/Docs/harbor_openshift_backend.md
+++ b/Docs/harbor_openshift_backend.md
@@ -1,7 +1,7 @@
 # Harbor OpenShift Backend — Handoff Document
 
 > **Jira:** APPENG-4906 (Phase 4 — Harbor OpenShift Backend)
-> **Target repo:** [GuyZivRH/skills_eval_corrections](https://github.com/GuyZivRH/skills_eval_corrections) (Harbor fork)
+> **Target repo:** [RHEcosystemAppEng/skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections) (Harbor fork)
 > **Full spec:** See [implementation_plan.md](./implementation_plan.md), Phase 4 (lines 270–343)
 
 ---
diff --git a/Docs/implementation_plan.md b/Docs/implementation_plan.md
index d6e7909..efc4f2d 100644
--- a/Docs/implementation_plan.md
+++ b/Docs/implementation_plan.md
@@ -26,7 +26,7 @@ The `tasks-treatment/` and `tasks-control/` directories generated during scaffol
 
 ### Harbor Fork
 
-The Harbor fork is at [GuyZivRH/skills_eval_corrections](https://github.com/GuyZivRH/skills_eval_corrections), forked from `harbor-framework/harbor`. The OpenShift backend will be added here as a new environment type alongside the existing GKE, Docker, Podman, Daytona, Modal, etc.
+The Harbor fork is at [RHEcosystemAppEng/skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections), forked from `harbor-framework/harbor`. The OpenShift backend will be added here as a new environment type alongside the existing GKE, Docker, Podman, Daytona, Modal, etc.
 
 ### LLM Access Strategy (Updated from ADR)
 
@@ -104,7 +104,7 @@ ABEvalFlow/
 | EventListener route/URL | Webhook target | Exposed via OpenShift Route; URL needed for GitHub webhook setup |
 | Submissions repo deploy key (read) | Clone submission on pipeline trigger | Stored as OpenShift Secret |
 | Quay.io registry | Image storage | Service account + push secret |
-| Harbor fork | Evaluation engine | [GuyZivRH/skills_eval_corrections](https://github.com/GuyZivRH/skills_eval_corrections) with OpenShift backend |
+| Harbor fork | Evaluation engine | [RHEcosystemAppEng/skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections) with OpenShift backend |
 | LLM access | Agent inference | One of: direct API key, opencode+self-hosted, or Vertex AI+LiteLLM |
 | PVC or MinIO (S3) | Artifact storage | For reports, logs, images |
 | agentic-collections deploy key (write) | Publish passing skills | Token/deploy key stored as OpenShift Secret |
@@ -269,7 +269,7 @@ Use digest-based references (not mutable tags) between tasks to avoid tag mutati
 
 ### 4.1 OpenShift Environment Backend (in Harbor fork)
 
-**Goal:** Create a new `OpenShiftEnvironment` class extending `BaseEnvironment` (from `src/harbor/environments/base.py`) in the [Harbor fork](https://github.com/GuyZivRH/skills_eval_corrections).
+**Goal:** Create a new `OpenShiftEnvironment` class extending `BaseEnvironment` (from `src/harbor/environments/base.py`) in the [Harbor fork](https://github.com/RHEcosystemAppEng/skills_eval_corrections).
 
 The GKE backend (`src/harbor/environments/gke.py`, ~1044 lines) serves as the reference implementation. The OpenShift backend must implement the full `BaseEnvironment` interface:
 
diff --git a/README.md b/README.md
index b3ba423..ae5894b 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ ABEvalFlow/
 | Repository | Purpose |
 |---|---|
 | [agentic-collections](https://github.com/RHEcosystemAppEng/agentic-collections) | Production skills, committed after evaluation passes |
-| [skills_eval_corrections](https://github.com/GuyZivRH/skills_eval_corrections) | Harbor fork with OpenShift backend |
+| [skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections) | Harbor fork with OpenShift backend |
 
 ## Submission Contract
 
diff --git a/scripts/generate_eval_config.py b/scripts/generate_eval_config.py
index 72bf39c..bcb76b6 100644
--- a/scripts/generate_eval_config.py
+++ b/scripts/generate_eval_config.py
@@ -152,10 +152,11 @@ def generate_eval_configs(
     metadata = load_metadata(submission_dir)
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    variant_args = {
-        "treatment": (treatment_task_dir, treatment_image_ref),
-        "control": (control_task_dir, control_image_ref),
-    }
+    variant_args = dict(zip(
+        VARIANTS,
+        ((treatment_task_dir, treatment_image_ref),
+         (control_task_dir, control_image_ref)),
+    ))
 
     configs: dict[str, dict[str, Any]] = {}
     for variant, (task_dir, img_ref) in variant_args.items():

From 04e81b0cc65031756d45b83c174f757d6995156e Mon Sep 17 00:00:00 2001
From: gziv <gziv@redhat.com>
Date: Thu, 23 Apr 2026 08:20:41 +0300
Subject: [PATCH 7/7] docs: align harbor handoff doc with actual implementation

Update harbor_openshift_backend.md to match the current state of the
Harbor fork (PRs #1 and #2):

- Document both eval modes (prebuilt + local-build via podman)
- Fix file path: openshift.py, not openshift_environment.py
- Correct Pod security: readOnlyRootFilesystem intentionally unset,
  HOME=/tmp injected instead
- Narrow RBAC table to actual usage (Pods, exec, log, Secrets, Events)
- Update naming to treatment/control throughout
- Add environment kwargs table (namespace, image_ref, registry,
  cpu_request, tls_verify)
- Document per-task environment_kwargs (fork PR #2)
- Document K8s client manager refactor for concurrent safety (fork PR #2)
- Update Definition of Done with completed items

Update harbor_fork_requirements.md to reflect per-task
environment_kwargs as implemented (no longer nice-to-have).
---
 Docs/harbor_fork_requirements.md |  18 ++--
 Docs/harbor_openshift_backend.md | 138 +++++++++++++++++++++----------
 2 files changed, 104 insertions(+), 52 deletions(-)

diff --git a/Docs/harbor_fork_requirements.md b/Docs/harbor_fork_requirements.md
index 0d34d4e..79bfb3d 100644
--- a/Docs/harbor_fork_requirements.md
+++ b/Docs/harbor_fork_requirements.md
@@ -138,22 +138,20 @@ ABEvalFlow's pass-rate parser expects:
 Where `reward > 0.0` means pass. This is Harbor's standard format — no change
 needed, but any deviation would break the parser.
 
-### Nice-to-have (not blocking)
+### Implemented (fork PR #2)
 
 **4. Per-task `environment_kwargs` support**
 
-ABEvalFlow currently runs each variant as a separate Harbor job to work around
-the global `environment.kwargs`. If we later want to run both variants in a
-single job (e.g., for sweep-based workflows), per-task `environment_kwargs`
-would be needed.
+Implemented in [PR #2](https://github.com/RHEcosystemAppEng/skills_eval_corrections/pull/2).
+`TaskConfig` now has an `environment_kwargs: dict[str, Any]` field. `Job._env_config_for_task`
+merges per-task kwargs into the global `EnvironmentConfig.kwargs` (task-level overrides global).
 
-Proposed change: add `environment_kwargs: dict[str, Any]` to `TaskConfig` in
-`src/harbor/models/trial/config.py`. When `Job` creates `TrialConfig` instances,
-merge per-task kwargs into the trial's `EnvironmentConfig.kwargs` (task-level
-overrides global).
+ABEvalFlow currently runs each variant as a separate Harbor job, which works
+without this feature. Per-task kwargs enables a single-job alternative for
+sweep-based workflows:
 
 ```yaml
-# Single-job example (requires this fork change):
+# Single-job example (supported since PR #2):
 tasks:
   - path: /workspace/tasks-treatment/my-submission
     environment_kwargs:
diff --git a/Docs/harbor_openshift_backend.md b/Docs/harbor_openshift_backend.md
index 96b08af..8f60dca 100644
--- a/Docs/harbor_openshift_backend.md
+++ b/Docs/harbor_openshift_backend.md
@@ -2,74 +2,113 @@
 
 > **Jira:** APPENG-4906 (Phase 4 — Harbor OpenShift Backend)
 > **Target repo:** [RHEcosystemAppEng/skills_eval_corrections](https://github.com/RHEcosystemAppEng/skills_eval_corrections) (Harbor fork)
-> **Full spec:** See [implementation_plan.md](./implementation_plan.md), Phase 4 (lines 270–343)
+> **Fork PRs:** [#1 — OpenShift environment backend](https://github.com/RHEcosystemAppEng/skills_eval_corrections/pull/1), [#2 — per-task environment_kwargs](https://github.com/RHEcosystemAppEng/skills_eval_corrections/pull/2)
+> **Full spec:** See [implementation_plan.md](./implementation_plan.md), Phase 4
 
 ---
 
-## What to Build
+## What Was Built
 
-A new `OpenShiftEnvironment` class in the Harbor fork that enables `harbor run --env openshift`. This backend manages trial Pod lifecycle on OpenShift using pre-built container images (built by ABEvalFlow's Tekton pipeline).
+An `OpenShiftEnvironment` class in the Harbor fork that enables
+`harbor run --env openshift`. This backend manages trial Pod lifecycle
+on OpenShift and supports two modes: pre-built container images (from
+the Tekton pipeline) and local build via podman.
 
 ## Where in the Harbor Fork
 
 | File | Action |
 |---|---|
-| `src/harbor/environments/openshift_environment.py` | Create — new backend |
-| `src/harbor/models/environment_type.py` | Edit — add `OPENSHIFT = "openshift"` to enum |
-| Backend registration (entry point or factory) | Edit — register so `--env openshift` works |
-| `tests/` | Create — unit tests with mocked K8s API |
+| `src/harbor/environments/openshift.py` | New backend |
+| `src/harbor/environments/k8s_client_manager.py` | Shared K8s client management (independent instances per caller) |
+| `src/harbor/models/environment_type.py` | `OPENSHIFT = "openshift"` added to enum |
+| `src/harbor/models/trial/config.py` | `environment_kwargs` field added to `TaskConfig` (PR #2) |
+| `src/harbor/job.py` | `_env_config_for_task` merges per-task kwargs into env config (PR #2) |
+| `tests/unit/environments/test_openshift.py` | Unit tests with mocked K8s API |
+| `tests/unit/models/test_trial_task_config.py` | Tests for per-task kwargs merge behavior |
+| `examples/configs/openshift-*.yaml` | Config examples for both modes + per-task kwargs |
 
 ## Reference Implementation
 
-Use the GKE backend as your template: `src/harbor/environments/gke.py` (~1044 lines). It implements the full `BaseEnvironment` interface from `src/harbor/environments/base.py`.
+The GKE backend (`src/harbor/environments/gke.py`) implements the full
+`BaseEnvironment` interface from `src/harbor/environments/base.py`.
 
 ## Key Differences from GKE
 
 ### `_init_client`
 - GKE: `gcloud container clusters get-credentials` + `load_kube_config()`
-- **OpenShift:** `load_incluster_config()` when running inside the cluster (Tekton), or `load_kube_config()` for local dev
+- **OpenShift:** `load_incluster_config()` when running inside the cluster
+  (Tekton), falls back to `load_kube_config()` for local dev
 
 ### `_build_and_push_image`
 - GKE: `gcloud builds submit` (Cloud Build)
-- **OpenShift: No-op.** Image build/push is handled by Tekton (Phase 3). The backend receives a digest-based image reference and only verifies the image exists and is pullable.
+- **OpenShift — two modes:**
 
-**Contract:**
+| Mode | Behavior | When to use |
+|------|----------|-------------|
+| **Prebuilt** (`image_ref` kwarg set) | Verify image is pullable, skip building | Default pipeline flow — Tekton builds with Buildah |
+| **Local build** (`force_build: true`) | Build with podman, push to specified registry | Local dev, skipping the build-push Tekton step |
+
+**Prebuilt contract:**
 - **Input:** Immutable image ref (e.g., `image-registry.openshift-image-registry.svc:5000/ab-eval-flow/my-skill@sha256:abc...`)
 - **Behavior:** Verify image exists using the trial ServiceAccount
 - **Output:** Return the image ref unchanged
-- **Failure:** Raise `ImageNotFoundError` / `ImageNotPullableError` with the image ref and SA identity
+- **Failure:** Raise `ImageNotFoundError` / `ImageNotPullableError`
 
-### `_image_exists`
-- GKE: `gcloud artifacts docker images describe`
-- **OpenShift:** Query internal registry API or use `skopeo inspect`
+**Local build contract:**
+- **Input:** `environment/Dockerfile` in the task directory, `registry` kwarg for push target
+- **Behavior:** `podman build` + `podman push` to the specified registry
+- **Kwargs:** `registry` (push target URL), `tls_verify` (default `"true"`)
 
 ### `start`, `stop`, `exec`, `upload_file/dir`, `download_file/dir`
 - Same K8s API patterns as GKE — the `kubernetes` Python client is identical
 
-## Pod Security Requirements
+## Environment kwargs
+
+Kwargs are passed via `environment.kwargs` in the job config YAML or
+via `--ek key=value` on the CLI. Per-task kwargs (PR #2) override
+global kwargs when set.
+
+| Kwarg | Description | Default |
+|-------|-------------|---------|
+| `namespace` | OpenShift namespace for trial Pods | Required |
+| `image_ref` | Digest-based image ref (prebuilt mode) | — |
+| `registry` | Push target URL (local-build mode) | — |
+| `cpu_request` | CPU request for trial Pods (e.g. `"250m"`) | — |
+| `tls_verify` | TLS verification for registry (e.g. `"false"`) | `"true"` |
+
+### Per-task environment_kwargs (PR #2)
 
-Trial Pods must run with hardened security context:
+`TaskConfig` now supports an `environment_kwargs` dict that merges into
+the global `environment.kwargs` (task-level overrides global). This
+enables treatment and control variants with different image refs in a
+single Harbor job:
 
 ```yaml
-securityContext:
-  runAsNonRoot: true
-  readOnlyRootFilesystem: true
-  allowPrivilegeEscalation: false
-  capabilities:
-    drop: ["ALL"]
-  seccompProfile:
-    type: RuntimeDefault
+tasks:
+  - path: /workspace/tasks-treatment/my-submission
+    environment_kwargs:
+      image_ref: "registry/ns/my-submission@sha256:abc..."
+  - path: /workspace/tasks-control/my-submission
+    environment_kwargs:
+      image_ref: "registry/ns/my-submission@sha256:def..."
 ```
 
-Since `readOnlyRootFilesystem: true` prevents writes, mount `emptyDir` volumes for:
-- `/tmp`
-- Agent cache directories (varies by agent)
+ABEvalFlow currently runs each variant as a separate Harbor job (two
+`harbor run` invocations). Per-task kwargs enables a single-job
+alternative if needed in the future.
+
+## Pod Security Requirements
+
+Trial Pods run with OpenShift's default security constraints. The
+`readOnlyRootFilesystem` is intentionally **not** set — many agent
+workloads need filesystem writes. Instead, `HOME=/tmp` is injected to
+direct writes to a writable location.
 
 Verify the target cluster uses `restricted-v2` SCC (OpenShift 4.11+).
 
 ## Trial Execution
 
-- **N = 20** attempts per variant (skilled + unskilled = 40 total sessions)
+- **N = 20** attempts per variant (treatment + control = 40 total sessions)
 - Image refs come as params from the build-push Tekton task (digest-based)
 - LLM endpoint via environment variable — backend is agnostic to LLM access mode
 - Configurable per-trial timeout and global evaluation timeout
@@ -82,37 +121,52 @@ The pipeline ServiceAccount in `ab-eval-flow` namespace needs:
 | Resource | Verbs | Purpose |
 |---|---|---|
 | Pods | create, get, list, watch, delete | Trial Pod lifecycle |
-| ConfigMaps | get, list | Trial configuration |
+| Pods/exec | create | Agent/verifier execution inside trial Pods |
+| Pods/log | get | Retrieving trial output |
 | Secrets | get | LLM credentials injection |
 | Events | get, list | Diagnosing hung/failed Pods |
-| PVCs | get, list, create | Pipeline workspaces |
-| ImageStreams | get, list | Registry access |
+
+## K8s Client Management
+
+The `BaseK8sClientManager` (shared by GKE and OpenShift backends) was
+refactored in PR #2 to return **independent `CoreV1Api` instances** per
+caller, each backed by its own `ApiClient`. This prevents concurrent
+`kubernetes.stream.stream()` calls (which monkey-patch `ApiClient.call_api`)
+from interfering with regular REST calls in other coroutines.
 
 ## Testing Strategy
 
 - **Unit tests:** Mock K8s API with `pytest` + `unittest.mock`. No live cluster needed.
-- **Integration tests:** Test against OpenShift developer sandbox (ROSA/OSD). Do **not** use Kind/Minikube — they won't catch SCC/Routes differences.
+- **Integration tests:** Test against OpenShift developer sandbox (ROSA/OSD).
+  Do **not** use Kind/Minikube — they won't catch SCC/Routes differences.
 
-## Tekton Task (in ABEvalFlow repo, not Harbor fork)
+## Tekton Task (in ABEvalFlow repo)
 
-A `pipeline/tasks/harbor-eval.yaml` Tekton Task will also be needed in ABEvalFlow to invoke Harbor. This task:
-- Accepts `skilled-image-ref` and `unskilled-image-ref` as params
-- Runs `harbor run --env openshift` with the image refs
-- Collects results to workspace/PVC
+The `harbor-eval` Tekton task (`pipeline/tasks/harbor-eval.yaml`) invokes
+Harbor from the ABEvalFlow pipeline:
 
-This can be built after the backend is functional.
+- Installs Harbor from the fork via `pip install git+<fork-url>@<revision>`
+- Generates two per-variant job configs using `scripts/generate_eval_config.py`
+- Runs `harbor run -c treatment-config.yaml` then `harbor run -c control-config.yaml`
+- Parses `result.json` files to compute pass rates as Tekton results
+- Supports `prebuilt` and `local-build` eval modes via a pipeline param
 
 ## Definition of Done
 
-- [ ] 40 trial Pods complete (20 skilled + 20 unskilled)
+- [x] `OpenShiftEnvironment` backend implemented (PR #1)
+- [x] Per-task `environment_kwargs` support (PR #2)
+- [x] Independent K8s client instances per caller (PR #2)
+- [x] Config examples for prebuilt, local-build, and per-task modes (PR #2)
+- [x] Unit tests with mocked K8s API
+- [ ] 40 trial Pods complete (20 treatment + 20 control) on live cluster
 - [ ] Cleanup verified — no stale Pods after evaluation
 - [ ] Retry behavior validated for transient failures
-- [ ] Unit tests pass with mocked K8s API
 - [ ] Integration test passes on OpenShift sandbox
 
 ## LLM Access Modes (for reference)
 
-The backend doesn't need to know which mode is used — it just passes env vars to trial Pods:
+The backend doesn't need to know which mode is used — it just passes
+env vars to trial Pods:
 
 | Mode | Env Var | Infrastructure |
 |---|---|---|