diff --git a/Docs/certification_and_checks.md b/Docs/certification_and_checks.md index 8aaf410..23130da 100644 --- a/Docs/certification_and_checks.md +++ b/Docs/certification_and_checks.md @@ -42,14 +42,16 @@ Levels are **hierarchical**: Certified requires Trusted, Trusted requires Founda ## Artifact Applicability Matrix -Different artifact types benefit from different checks: - -| Artifact Type | Focus Area | Key Checks | -|---------------|------------|------------| -| **Skills** (Prompt + Code) | A/B gap testing, execution isolation | Functional Validation, Execution Validation | -| **MCP Servers** (Tool APIs) | API contracts, schema accuracy | Resilience/Chaos Testing, Operational Policy | -| **Plugins** (REST APIs) | OpenAPI validation, auth flows | Security Validation, Metadata Compliance | -| **Agent Evals** (Autonomous) | Reasoning trace, multi-turn | Advanced Agent Validation, Safety Guardrails | +Different artifact types benefit from different checks. Each artifact type has a dedicated certification profile (see `config/certification_profiles.yaml`) that defines which checks apply at each level. + +| Artifact Type | Focus Area | Foundational | Trusted | Certified | +|---------------|------------|--------------|---------|-----------| +| **Skills** (Prompt + Code) | A/B gap testing, execution isolation | Structure, Security, Execution, Quality, Metadata | Eval Assets, Functional, Instruction Quality | Enterprise Structure, Advanced Agent | +| **MCP Servers** (Tool APIs) | API contracts, operational stability | Structure, Security, Metadata | Eval Assets, Functional | Enterprise Structure, Enterprise Security | +| **Plugins** (REST APIs) | OpenAPI validation, auth flows | Structure, Security, Metadata | Eval Assets, Advanced Security, Functional | Enterprise Structure, Enterprise Security | +| **Agent Evals** (Autonomous) | Reasoning, safety, multi-turn | Structure, Security, Execution, Metadata | Eval Assets, Functional, Instruction Quality | Enterprise Structure, Advanced Agent | + +**Profile selection** is done at pipeline deployment level via `--certification-profile=`. Default is `skill`. --- diff --git a/abevalflow/certification.py b/abevalflow/certification.py index 82ccc14..5d0bd11 100644 --- a/abevalflow/certification.py +++ b/abevalflow/certification.py @@ -122,7 +122,7 @@ def _load_profiles_yaml(profiles_path: Path | None = None) -> dict[str, Any]: path = profiles_path or DEFAULT_PROFILES_PATH if not path.exists(): raise FileNotFoundError(f"Certification profiles not found: {path}") - + with open(path) as f: return yaml.safe_load(f) @@ -156,7 +156,7 @@ def get_default_profile_name(profiles_path: Path | None = None) -> str: def load_profile( profile_name: str | None = None, profiles_path: Path | None = None, -) -> "CertificationPolicy": +) -> CertificationPolicy: """Load a certification profile and return it as a CertificationPolicy. Profiles define which checks apply to each certification level for different @@ -183,41 +183,39 @@ def load_profile( data = _load_profiles_yaml(profiles_path) profiles = data.get("profiles", {}) - + name = profile_name or data.get("default_profile", "skill") - + if name not in profiles: available = list(profiles.keys()) - raise ValueError( - f"Unknown certification profile '{name}'. Available: {available}" - ) - + raise ValueError(f"Unknown certification profile '{name}'. Available: {available}") + profile_data = profiles[name] logger.info("Loading certification profile '%s': %s", name, profile_data.get("description", "")) - + # Build CertificationPolicy from profile data foundational = None trusted = None certified = None - + if "foundational" in profile_data: foundational = CertificationLevelPolicy( checks=profile_data["foundational"].get("checks"), thresholds=profile_data["foundational"].get("thresholds"), ) - + if "trusted" in profile_data: trusted = CertificationLevelPolicy( checks=profile_data["trusted"].get("checks"), thresholds=profile_data["trusted"].get("thresholds"), ) - + if "certified" in profile_data: certified = CertificationLevelPolicy( checks=profile_data["certified"].get("checks"), thresholds=profile_data["certified"].get("thresholds"), ) - + return CertificationPolicy( foundational=foundational, trusted=trusted, @@ -370,9 +368,7 @@ def _map_gate_to_checks( details={"source_implementation": source_impl}, ) ) - advanced_threshold = _get_threshold( - CheckId.ADVANCED_AGENT_VALIDATION, threshold_overrides - ) + advanced_threshold = _get_threshold(CheckId.ADVANCED_AGENT_VALIDATION, threshold_overrides) if gate.score >= advanced_threshold: checks.append( CheckResult( @@ -413,9 +409,7 @@ def _map_gate_to_checks( }, ) ) - adv_security_threshold = _get_threshold( - CheckId.ADVANCED_SECURITY_VALIDATION, threshold_overrides - ) + adv_security_threshold = _get_threshold(CheckId.ADVANCED_SECURITY_VALIDATION, threshold_overrides) high_score = gate.score >= adv_security_threshold and gate.passed checks.append( CheckResult( @@ -459,9 +453,7 @@ def _map_gate_to_checks( details={"source_implementation": source_impl}, ) ) - instruction_threshold = _get_threshold( - CheckId.INSTRUCTION_QUALITY, threshold_overrides - ) + instruction_threshold = _get_threshold(CheckId.INSTRUCTION_QUALITY, threshold_overrides) checks.append( CheckResult( check_id=CheckId.INSTRUCTION_QUALITY, @@ -493,15 +485,13 @@ def _validate_check_ids(check_ids: list[str]) -> list[CheckId]: result = [] for check_id in check_ids: if check_id not in valid_ids: - raise ValueError( - f"Invalid check ID '{check_id}'. Valid IDs are: {sorted(valid_ids)}" - ) + raise ValueError(f"Invalid check ID '{check_id}'. Valid IDs are: {sorted(valid_ids)}") result.append(CheckId(check_id)) return result def _collect_threshold_overrides( - policy: "CertificationPolicy | None", + policy: CertificationPolicy | None, ) -> dict[str, float]: """Collect all threshold overrides from a certification policy.""" if policy is None: @@ -519,7 +509,7 @@ def compute_certification( validation_passed: bool = True, metadata_valid: bool = True, has_eval_assets: bool = True, - policy: "CertificationPolicy | None" = None, + policy: CertificationPolicy | None = None, ) -> CertificationResult: """Compute certification levels from gate results. @@ -664,15 +654,9 @@ def build_level( return LevelResult(level=level, passed=all_passed, checks=level_checks) # Build all levels - foundational_result = build_level( - CertificationLevel.FOUNDATIONAL, "foundational", FOUNDATIONAL_CHECKS - ) - trusted_result = build_level( - CertificationLevel.TRUSTED, "trusted", TRUSTED_CHECKS - ) - certified_result = build_level( - CertificationLevel.CERTIFIED, "certified", CERTIFIED_CHECKS - ) + foundational_result = build_level(CertificationLevel.FOUNDATIONAL, "foundational", FOUNDATIONAL_CHECKS) + trusted_result = build_level(CertificationLevel.TRUSTED, "trusted", TRUSTED_CHECKS) + certified_result = build_level(CertificationLevel.CERTIFIED, "certified", CERTIFIED_CHECKS) # Enforce hierarchy: lower levels must pass for higher levels to pass # If foundational fails, trusted and certified cannot pass diff --git a/abevalflow/compass_facts.py b/abevalflow/compass_facts.py index 74e91b4..f5bba31 100644 --- a/abevalflow/compass_facts.py +++ b/abevalflow/compass_facts.py @@ -23,7 +23,7 @@ import re import urllib.error import urllib.request -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import TYPE_CHECKING, Any from pydantic import BaseModel, Field @@ -65,13 +65,13 @@ class FactPushResult(BaseModel): success: bool = Field(description="Whether the push succeeded") error: str | None = Field(default=None, description="Error message if push failed") pushed_at: datetime = Field( - default_factory=lambda: datetime.now(timezone.utc), + default_factory=lambda: datetime.now(UTC), description="Timestamp of the push attempt", ) def _build_fact_payload( - gate_result: "GateResult", + gate_result: GateResult, entity_ref: str, fact_ref: str, ) -> dict[str, Any]: @@ -100,7 +100,7 @@ def _build_fact_payload( "evaluated_at": ( gate_result.evaluated_at.isoformat() if gate_result.evaluated_at - else datetime.now(timezone.utc).isoformat() + else datetime.now(UTC).isoformat() ), }, } @@ -109,7 +109,7 @@ def _build_fact_payload( def push_gate_fact( - gate_result: "GateResult", + gate_result: GateResult, endpoint: str, entity_ref: str, fact_ref: str, @@ -139,7 +139,7 @@ def push_gate_fact( } if bearer_token: headers["Authorization"] = f"Bearer {bearer_token}" - + request = urllib.request.Request( endpoint, data=data, @@ -210,8 +210,8 @@ def push_gate_fact( def push_gate_fact_from_config( - gate_result: "GateResult", - push_facts_config: "PushFactsConfig", + gate_result: GateResult, + push_facts_config: PushFactsConfig, ) -> FactPushResult: """Push a gate result using PushFactsConfig settings. @@ -250,7 +250,7 @@ def push_gate_fact_from_config( def validate_push_facts_config( - push_facts_config: "PushFactsConfig | None", + push_facts_config: PushFactsConfig | None, gates_with_push_fact: list[str], ) -> None: """Validate push_facts configuration and warn if misconfigured. @@ -280,13 +280,13 @@ class CertificationFactPushResult(BaseModel): success: bool = Field(description="Whether the push succeeded") error: str | None = Field(default=None, description="Error message if push failed") pushed_at: datetime = Field( - default_factory=lambda: datetime.now(timezone.utc), + default_factory=lambda: datetime.now(UTC), description="Timestamp of the push attempt", ) def _build_certification_level_payload( - level_result: "LevelResult", + level_result: LevelResult, entity_ref: str, fact_ref: str, ) -> dict[str, Any]: @@ -302,14 +302,16 @@ def _build_certification_level_payload( """ checks_data = [] for check in level_result.checks: - checks_data.append({ - "check_id": check.check_id.value, - "name": check.name, - "passed": check.passed, - "score": check.score, - "message": check.message, - "source_gate": check.source_gate, - }) + checks_data.append( + { + "check_id": check.check_id.value, + "name": check.name, + "passed": check.passed, + "score": check.score, + "message": check.message, + "source_gate": check.source_gate, + } + ) # Determine if this level failed due to hierarchy (all checks passed but level failed) all_checks_passed = level_result.checks_total > 0 and level_result.checks_passed == level_result.checks_total @@ -322,7 +324,7 @@ def _build_certification_level_payload( "checks_total": level_result.checks_total, "overall_score": level_result.overall_score, "checks": checks_data, - "evaluated_at": datetime.now(timezone.utc).isoformat(), + "evaluated_at": datetime.now(UTC).isoformat(), } if hierarchy_forced_failure: @@ -340,7 +342,7 @@ def _build_certification_level_payload( def _build_certification_summary_payload( - certification_result: "CertificationResult", + certification_result: CertificationResult, entity_ref: str, fact_ref: str, ) -> dict[str, Any]: @@ -371,7 +373,7 @@ def _build_certification_summary_payload( "trusted_score": certification_result.trusted.overall_score, "certified_passed": certification_result.certified.passed, "certified_score": certification_result.certified.overall_score, - "evaluated_at": datetime.now(timezone.utc).isoformat(), + "evaluated_at": datetime.now(UTC).isoformat(), }, } ] @@ -379,7 +381,7 @@ def _build_certification_summary_payload( def push_certification_level_fact( - level_result: "LevelResult", + level_result: LevelResult, endpoint: str, entity_ref: str, fact_ref: str, @@ -534,8 +536,8 @@ def _push_raw_fact( def push_certification_facts( - certification_result: "CertificationResult", - push_facts_config: "PushFactsConfig", + certification_result: CertificationResult, + push_facts_config: PushFactsConfig, ) -> list[CertificationFactPushResult]: """Push all certification level facts to Compass. diff --git a/abevalflow/db/engine.py b/abevalflow/db/engine.py index 0106f6f..6330522 100644 --- a/abevalflow/db/engine.py +++ b/abevalflow/db/engine.py @@ -6,8 +6,8 @@ import os from sqlalchemy import Engine, create_engine -from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session, sessionmaker from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential from abevalflow.db.models import Base diff --git a/abevalflow/db/models.py b/abevalflow/db/models.py index 35708a9..85240a1 100644 --- a/abevalflow/db/models.py +++ b/abevalflow/db/models.py @@ -9,7 +9,7 @@ from __future__ import annotations import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from sqlalchemy import ( Boolean, @@ -32,7 +32,7 @@ class Base(DeclarativeBase): def _utcnow() -> datetime: - return datetime.now(timezone.utc) + return datetime.now(UTC) # Portable JSON that upgrades to JSONB on PostgreSQL @@ -44,22 +44,16 @@ class EvaluationRun(Base): __tablename__ = "evaluation_runs" - id: Mapped[uuid.UUID] = mapped_column( - Uuid, primary_key=True, default=uuid.uuid4 - ) + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) submission_name: Mapped[str] = mapped_column(String(255), nullable=False) - pipeline_run_id: Mapped[str] = mapped_column( - String(255), unique=True, nullable=False - ) + pipeline_run_id: Mapped[str] = mapped_column(String(255), unique=True, nullable=False) # Provenance commit_sha: Mapped[str | None] = mapped_column(String(64)) treatment_image_ref: Mapped[str | None] = mapped_column(Text) control_image_ref: Mapped[str | None] = mapped_column(Text) harbor_fork_revision: Mapped[str | None] = mapped_column(String(64)) - eval_engine: Mapped[str] = mapped_column( - String(10), nullable=False, default="harbor" - ) + eval_engine: Mapped[str] = mapped_column(String(10), nullable=False, default="harbor") # Summary recommendation: Mapped[str] = mapped_column(String(10), nullable=False) @@ -91,13 +85,9 @@ class EvaluationRun(Base): # Full report for flexibility / future queries report_json: Mapped[dict] = mapped_column(_JsonVariant, nullable=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=_utcnow - ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) - trials: Mapped[list[Trial]] = relationship( - back_populates="run", cascade="all, delete-orphan" - ) + trials: Mapped[list[Trial]] = relationship(back_populates="run", cascade="all, delete-orphan") __table_args__ = ( Index("ix_evaluation_runs_submission_name", "submission_name"), @@ -109,10 +99,7 @@ class EvaluationRun(Base): ) def __repr__(self) -> str: - return ( - f"" - ) + return f"" class Trial(Base): @@ -120,9 +107,7 @@ class Trial(Base): __tablename__ = "trials" - id: Mapped[uuid.UUID] = mapped_column( - Uuid, primary_key=True, default=uuid.uuid4 - ) + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) run_id: Mapped[uuid.UUID] = mapped_column( Uuid, ForeignKey("evaluation_runs.id", ondelete="CASCADE"), nullable=False ) @@ -131,9 +116,7 @@ class Trial(Base): reward: Mapped[float | None] = mapped_column(Float) passed: Mapped[bool] = mapped_column(Boolean, nullable=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=_utcnow - ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) run: Mapped[EvaluationRun] = relationship(back_populates="trials") @@ -143,10 +126,7 @@ class Trial(Base): ) def __repr__(self) -> str: - return ( - f"" - ) + return f"" class SecurityScan(Base): @@ -158,12 +138,8 @@ class SecurityScan(Base): __tablename__ = "security_scans" - id: Mapped[uuid.UUID] = mapped_column( - Uuid, primary_key=True, default=uuid.uuid4 - ) - pipeline_run_id: Mapped[str] = mapped_column( - String(255), nullable=False, index=True - ) + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) + pipeline_run_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) submission_name: Mapped[str] = mapped_column(String(255), nullable=False) scanner: Mapped[str] = mapped_column(String(50), nullable=False) @@ -178,9 +154,7 @@ class SecurityScan(Base): findings_json: Mapped[list] = mapped_column(_JsonVariant, nullable=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=_utcnow - ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) __table_args__ = ( Index("ix_security_scans_submission_name", "submission_name"), @@ -207,12 +181,8 @@ class MCPCheckerRun(Base): __tablename__ = "mcpchecker_results" - id: Mapped[uuid.UUID] = mapped_column( - Uuid, primary_key=True, default=uuid.uuid4 - ) - pipeline_run_id: Mapped[str] = mapped_column( - String(255), unique=True, nullable=False - ) + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) + pipeline_run_id: Mapped[str] = mapped_column(String(255), unique=True, nullable=False) submission_name: Mapped[str] = mapped_column(String(255), nullable=False) eval_name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -227,13 +197,9 @@ class MCPCheckerRun(Base): raw_output_json: Mapped[dict] = mapped_column(_JsonVariant, nullable=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=_utcnow - ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) - tasks: Mapped[list[MCPCheckerTask]] = relationship( - back_populates="run", cascade="all, delete-orphan" - ) + tasks: Mapped[list[MCPCheckerTask]] = relationship(back_populates="run", cascade="all, delete-orphan") __table_args__ = ( Index("ix_mcpchecker_results_submission_name", "submission_name"), @@ -256,9 +222,7 @@ class MCPCheckerTask(Base): __tablename__ = "mcpchecker_tasks" - id: Mapped[uuid.UUID] = mapped_column( - Uuid, primary_key=True, default=uuid.uuid4 - ) + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) run_id: Mapped[uuid.UUID] = mapped_column( Uuid, ForeignKey("mcpchecker_results.id", ondelete="CASCADE"), nullable=False ) @@ -275,9 +239,7 @@ class MCPCheckerTask(Base): details_json: Mapped[dict | None] = mapped_column(_JsonVariant, nullable=True) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=_utcnow - ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) run: Mapped[MCPCheckerRun] = relationship(back_populates="tasks") diff --git a/abevalflow/engines/__init__.py b/abevalflow/engines/__init__.py index cc2e81a..3cd86c3 100644 --- a/abevalflow/engines/__init__.py +++ b/abevalflow/engines/__init__.py @@ -16,9 +16,11 @@ def register_engine(name: str): """Decorator to register an engine class.""" + def decorator(cls: type[EvalEngine]) -> type[EvalEngine]: _ENGINE_REGISTRY[name] = cls return cls + return decorator @@ -45,9 +47,9 @@ def get_all_engines() -> list[str]: return list(_ENGINE_REGISTRY.keys()) -from abevalflow.engines.harbor import HarborEngine -from abevalflow.engines.ase import ASEEngine from abevalflow.engines.a2a import A2AEngine +from abevalflow.engines.ase import ASEEngine +from abevalflow.engines.harbor import HarborEngine from abevalflow.engines.mcpchecker import MCPCheckerEngine __all__ = [ diff --git a/abevalflow/engines/mcpchecker.py b/abevalflow/engines/mcpchecker.py index 7cbc4c7..d88d7fe 100644 --- a/abevalflow/engines/mcpchecker.py +++ b/abevalflow/engines/mcpchecker.py @@ -9,7 +9,7 @@ from abevalflow.engines import register_engine from abevalflow.engines.base import EvalEngine -from abevalflow.gates.base import Finding, GateMode, GateResult, GateType, Severity +from abevalflow.gates.base import Finding, GateResult, GateType, Severity from abevalflow.schemas import GatePolicy logger = logging.getLogger(__name__) @@ -50,11 +50,11 @@ def to_gate_result( overall_score = raw_result.get("overall_score", 0.0) passed_tasks = raw_result.get("passed_tasks", 0) - failed_tasks = raw_result.get("failed_tasks", 0) - error_tasks = raw_result.get("error_tasks", 0) + _failed_tasks = raw_result.get("failed_tasks", 0) # Reserved for future use + _error_tasks = raw_result.get("error_tasks", 0) # Reserved for future use total_tasks = raw_result.get("total_tasks", 0) - recommendation = raw_result.get("recommendation", "fail") + _recommendation = raw_result.get("recommendation", "fail") # Reserved for future use passed = overall_score >= threshold findings = [] @@ -62,12 +62,14 @@ def to_gate_result( status = task.get("status", "unknown") if status in ("failed", "error"): severity = Severity.HIGH if status == "error" else Severity.MEDIUM - findings.append(Finding( - severity=severity, - message=task.get("error_message") or f"Task {status}: {task.get('task_name', 'unknown')}", - location=task.get("task_id"), - rule_id=f"mcpchecker-{status}", - )) + findings.append( + Finding( + severity=severity, + message=task.get("error_message") or f"Task {status}: {task.get('task_name', 'unknown')}", + location=task.get("task_id"), + rule_id=f"mcpchecker-{status}", + ) + ) message = ( f"MCPChecker: {passed_tasks}/{total_tasks} tasks passed " diff --git a/abevalflow/experiment.py b/abevalflow/experiment.py index da7a490..029581d 100644 --- a/abevalflow/experiment.py +++ b/abevalflow/experiment.py @@ -18,7 +18,9 @@ class ExperimentStrategy(Protocol): """Protocol for experiment variant strategies.""" def variant_copy_specs( - self, submission_dir: Path, variant: str, + self, + submission_dir: Path, + variant: str, ) -> list[CopySpec]: """Return copy specs for this variant ('control' or 'treatment'). @@ -27,7 +29,10 @@ def variant_copy_specs( ... def customize_context( - self, base_context: dict, variant: str, submission_dir: Path, + self, + base_context: dict, + variant: str, + submission_dir: Path, ) -> dict: """Adjust template context per variant. @@ -67,13 +72,18 @@ def __init__(self, config: ExperimentConfig) -> None: self._config = config def variant_copy_specs( - self, submission_dir: Path, variant: str, + self, + submission_dir: Path, + variant: str, ) -> list[CopySpec]: specs = _get_variant_spec(self._config, variant).copy_dirs return _filter_specs(specs, submission_dir) def customize_context( - self, base_context: dict, variant: str, submission_dir: Path, + self, + base_context: dict, + variant: str, + submission_dir: Path, ) -> dict: filtered = self.variant_copy_specs(submission_dir, variant) ctx = {**base_context} @@ -94,13 +104,18 @@ def __init__(self, config: ExperimentConfig) -> None: self._config = config def variant_copy_specs( - self, submission_dir: Path, variant: str, + self, + submission_dir: Path, + variant: str, ) -> list[CopySpec]: specs = _get_variant_spec(self._config, variant).copy_dirs return _filter_specs(specs, submission_dir) def customize_context( - self, base_context: dict, variant: str, submission_dir: Path, + self, + base_context: dict, + variant: str, + submission_dir: Path, ) -> dict: variant_spec = _get_variant_spec(self._config, variant) filtered = _filter_specs(variant_spec.copy_dirs, submission_dir) diff --git a/abevalflow/gates/base.py b/abevalflow/gates/base.py index 0592f8b..faff9ba 100644 --- a/abevalflow/gates/base.py +++ b/abevalflow/gates/base.py @@ -9,7 +9,7 @@ Each gate produces a GateResult with a normalized score and pass/fail status. """ -from datetime import datetime, timezone +from datetime import UTC, datetime from enum import StrEnum from typing import Any @@ -80,9 +80,7 @@ class GateResult(BaseModel): """ gate_type: GateType = Field(description="Type of gate: engine, security, or quality") - gate_name: str = Field( - description="Category name of the gate (e.g., 'evaluation', 'security', 'quality')" - ) + gate_name: str = Field(description="Category name of the gate (e.g., 'evaluation', 'security', 'quality')") policy_key: str | None = Field( default=None, description="Implementation name for policy lookup (e.g., 'harbor', 'cisco'). " @@ -115,7 +113,7 @@ class GateResult(BaseModel): description="Human-readable summary of the gate result", ) evaluated_at: datetime = Field( - default_factory=lambda: datetime.now(timezone.utc), + default_factory=lambda: datetime.now(UTC), description="Timestamp when the gate was evaluated", ) diff --git a/abevalflow/gates/quality/__init__.py b/abevalflow/gates/quality/__init__.py index a4ecb11..ff227f0 100644 --- a/abevalflow/gates/quality/__init__.py +++ b/abevalflow/gates/quality/__init__.py @@ -12,9 +12,11 @@ def register_quality_gate(name: str): """Decorator to register a quality gate class.""" + def decorator(cls: type[QualityGate]) -> type[QualityGate]: _QUALITY_GATE_REGISTRY[name] = cls return cls + return decorator diff --git a/abevalflow/gates/quality/llm_review.py b/abevalflow/gates/quality/llm_review.py index 6d6c016..4d50e46 100644 --- a/abevalflow/gates/quality/llm_review.py +++ b/abevalflow/gates/quality/llm_review.py @@ -126,13 +126,15 @@ def evaluate( severity = Severity.INFO if dim_score < 0.6 and dim_finding: - findings.append(Finding( - severity=severity, - message=f"{dim_name}: {dim_finding}", - location=dim_name, - rule_id=f"quality-{dim_name}", - details={"score": dim_score}, - )) + findings.append( + Finding( + severity=severity, + message=f"{dim_name}: {dim_finding}", + location=dim_name, + rule_id=f"quality-{dim_name}", + details={"score": dim_score}, + ) + ) # In BLOCK mode, threshold is authoritative for pass/fail # In WARN mode, only hard "fail" recommendations fail the gate @@ -143,10 +145,7 @@ def evaluate( passed = recommendation != "fail" summary = review_data.get("summary", "") - dimension_scores = { - name: data.get("score", 0.0) - for name, data in dimensions.items() - } + dimension_scores = {name: data.get("score", 0.0) for name, data in dimensions.items()} comparison = ">=" if passed else "<" message = ( diff --git a/abevalflow/gates/security/__init__.py b/abevalflow/gates/security/__init__.py index 1decbb2..8e9f1d6 100644 --- a/abevalflow/gates/security/__init__.py +++ b/abevalflow/gates/security/__init__.py @@ -12,9 +12,11 @@ def register_security_gate(name: str): """Decorator to register a security gate class.""" + def decorator(cls: type[SecurityGate]) -> type[SecurityGate]: _SECURITY_GATE_REGISTRY[name] = cls return cls + return decorator diff --git a/abevalflow/gates/security/cisco.py b/abevalflow/gates/security/cisco.py index 1df8ca6..3d1b945 100644 --- a/abevalflow/gates/security/cisco.py +++ b/abevalflow/gates/security/cisco.py @@ -104,13 +104,15 @@ def evaluate( if severity.value in severity_counts: severity_counts[severity.value] += 1 - findings.append(Finding( - severity=severity, - message=f.get("message", f.get("description", "")), - location=f.get("file_path", f.get("location", {}).get("file")), - rule_id=f.get("rule_id", f.get("id", "unknown")), - details=f, - )) + findings.append( + Finding( + severity=severity, + message=f.get("message", f.get("description", "")), + location=f.get("file_path", f.get("location", {}).get("file")), + rule_id=f.get("rule_id", f.get("id", "unknown")), + details=f, + ) + ) high_or_critical = severity_counts["critical"] + severity_counts["high"] @@ -124,11 +126,11 @@ def evaluate( score = 1.0 else: weighted_score = ( - severity_counts["critical"] * 0.0 + - severity_counts["high"] * 0.25 + - severity_counts["medium"] * 0.5 + - severity_counts["low"] * 0.75 + - severity_counts["info"] * 0.9 + severity_counts["critical"] * 0.0 + + severity_counts["high"] * 0.25 + + severity_counts["medium"] * 0.5 + + severity_counts["low"] * 0.75 + + severity_counts["info"] * 0.9 ) score = weighted_score / total_findings if total_findings > 0 else 1.0 diff --git a/abevalflow/generation_validator.py b/abevalflow/generation_validator.py index feaaff2..360efda 100644 --- a/abevalflow/generation_validator.py +++ b/abevalflow/generation_validator.py @@ -58,6 +58,7 @@ def _parse_json_or_text(response_text: str, label: str = "Review") -> dict: logger.warning("%s ambiguous, treating as pass: %s", label, response_text[:200]) return {"passed": True, "issues": []} + CONTENT_CHECK_SYSTEM_PROMPT = """\ You are a QA gate for AI-generated skill evaluation files. @@ -120,10 +121,7 @@ def scenario_coherence_check( if basename not in instruction: missing_in_instruction.append(filepath) if missing_in_instruction: - errors.append( - f"Instruction is missing project files from scenario brief: " - f"{missing_in_instruction}" - ) + errors.append(f"Instruction is missing project files from scenario brief: {missing_in_instruction}") missing_in_tests = [] for field, value in expected.items(): @@ -133,10 +131,7 @@ def scenario_coherence_check( if val_str not in test_content: missing_in_tests.append(f"{field}={val_str}") if missing_in_tests: - errors.append( - f"Tests are missing expected_outputs from scenario brief: " - f"{missing_in_tests}" - ) + errors.append(f"Tests are missing expected_outputs from scenario brief: {missing_in_tests}") missing_in_instruction_outputs = [] for field, value in expected.items(): @@ -146,10 +141,7 @@ def scenario_coherence_check( if val_str not in instruction: missing_in_instruction_outputs.append(f"{field}={val_str}") if missing_in_instruction_outputs: - errors.append( - f"Instruction is missing expected_outputs from scenario brief: " - f"{missing_in_instruction_outputs}" - ) + errors.append(f"Instruction is missing expected_outputs from scenario brief: {missing_in_instruction_outputs}") return errors @@ -264,7 +256,8 @@ def content_check(submission_dir: Path) -> dict: Be strict but pragmatic — minor style issues are acceptable.""" -_COVERAGE_REVIEWER_SYSTEM = """\ +_COVERAGE_REVIEWER_SYSTEM = ( + """\ You are a **test coverage reviewer** for AI-generated skill evaluation files. You will receive a skill definition (SKILL.md), a generated task instruction \ @@ -275,9 +268,12 @@ def content_check(submission_dir: Path) -> dict: 2. Are edge cases and error paths tested? 3. Are there any untested requirements? 4. Do the tests actually assert meaningful outcomes (not just "assert True")? -""" + _REVIEWER_JSON_INSTRUCTION +""" + + _REVIEWER_JSON_INSTRUCTION +) -_ALIGNMENT_REVIEWER_SYSTEM = """\ +_ALIGNMENT_REVIEWER_SYSTEM = ( + """\ You are a **skill alignment reviewer** for AI-generated skill evaluation files. You will receive a skill definition (SKILL.md), a generated task instruction \ @@ -300,9 +296,12 @@ def content_check(submission_dir: Path) -> dict: hardcoded scenario with no room for the agent to demonstrate genuine \ understanding? Tests should verify correct behavior, not just pattern-match \ against pre-embedded answers from the instruction. -""" + _REVIEWER_JSON_INSTRUCTION +""" + + _REVIEWER_JSON_INSTRUCTION +) -_FEASIBILITY_REVIEWER_SYSTEM = """\ +_FEASIBILITY_REVIEWER_SYSTEM = ( + """\ You are a **feasibility reviewer** for AI-generated skill evaluation files. You will receive a skill definition (SKILL.md), a generated task instruction \ @@ -313,7 +312,9 @@ def content_check(submission_dir: Path) -> dict: 2. Are the tests deterministic (no randomness, timing, or external dependencies)? 3. Do the tests import from /workspace correctly? 4. Are there any circular or impossible requirements? -""" + _REVIEWER_JSON_INSTRUCTION +""" + + _REVIEWER_JSON_INSTRUCTION +) _REVIEWERS = [ ("coverage", _COVERAGE_REVIEWER_SYSTEM), @@ -323,7 +324,9 @@ def content_check(submission_dir: Path) -> dict: def _run_single_review( - reviewer_name: str, system_prompt: str, user_prompt: str, + reviewer_name: str, + system_prompt: str, + user_prompt: str, ) -> tuple[str, dict]: """Execute a single LLM reviewer call. Returns (name, result_dict).""" response_text = llm_client.chat_completion( @@ -479,7 +482,8 @@ def _read_safe(path: Path) -> str: If there are problems, list them: {"pass": false, "issues": ["specific issue"]} """ -_ASE_SKILL_SPECIFICITY_REVIEWER = """\ +_ASE_SKILL_SPECIFICITY_REVIEWER = ( + """\ You are a reviewer checking whether ASE eval assertions test SKILL-SPECIFIC knowledge. Your job: verify that the assertions differentiate a skill-enhanced LLM from a generic one. @@ -493,9 +497,12 @@ def _read_safe(path: Path) -> str: - Assertions test only generic LLM capabilities (e.g., "responds politely") - Any LLM could pass without reading the skill document - Assertions are too vague to meaningfully evaluate skill knowledge -""" + _ASE_REVIEWER_JSON_INSTRUCTION +""" + + _ASE_REVIEWER_JSON_INSTRUCTION +) -_ASE_PROMPT_QUALITY_REVIEWER = """\ +_ASE_PROMPT_QUALITY_REVIEWER = ( + """\ You are a reviewer checking whether ASE eval prompts are clear and testable. Your job: verify that prompts will elicit skill-relevant responses. @@ -509,9 +516,12 @@ def _read_safe(path: Path) -> str: - Prompts are vague or could be interpreted multiple ways - Prompts don't relate to the skill's core functionality - Prompts are too complex or combine too many concepts -""" + _ASE_REVIEWER_JSON_INSTRUCTION +""" + + _ASE_REVIEWER_JSON_INSTRUCTION +) -_ASE_ASSERTION_ALIGNMENT_REVIEWER = """\ +_ASE_ASSERTION_ALIGNMENT_REVIEWER = ( + """\ You are a reviewer checking whether ASE assertions align with the expected_output. Your job: verify assertions match what a correct response should contain. @@ -525,7 +535,9 @@ def _read_safe(path: Path) -> str: - Assertions don't match what expected_output describes - Assertions are redundant or overlap significantly - Assertions would pass for incorrect responses -""" + _ASE_REVIEWER_JSON_INSTRUCTION +""" + + _ASE_REVIEWER_JSON_INSTRUCTION +) _ASE_REVIEWERS = [ ("skill_specificity", _ASE_SKILL_SPECIFICITY_REVIEWER), diff --git a/abevalflow/harbor_agents/a2a_adapter.py b/abevalflow/harbor_agents/a2a_adapter.py index d4dd3e9..c5be797 100644 --- a/abevalflow/harbor_agents/a2a_adapter.py +++ b/abevalflow/harbor_agents/a2a_adapter.py @@ -19,7 +19,6 @@ from __future__ import annotations -import asyncio import json import logging import uuid @@ -141,14 +140,10 @@ async def run( try: response_data = await self._send_request(payload) - await self._process_response( - response_data, environment, context, instruction - ) - except asyncio.TimeoutError: + await self._process_response(response_data, environment, context, instruction) + except TimeoutError: self.logger.error(f"A2A request timed out after {self.timeout}s") - await self._write_error_response( - environment, f"Request timed out after {self.timeout} seconds" - ) + await self._write_error_response(environment, f"Request timed out after {self.timeout} seconds") raise except aiohttp.ClientError as e: self.logger.error(f"A2A request failed: {e}") @@ -207,9 +202,7 @@ async def _process_response( state = status.get("state", "unknown") if state == "failed": - error_msg = status.get("message", {}).get("parts", [{}])[0].get( - "text", "Agent task failed" - ) + error_msg = status.get("message", {}).get("parts", [{}])[0].get("text", "Agent task failed") self.logger.error(f"A2A agent task failed: {error_msg}") agent_response_text = self._extract_response_text(result) @@ -232,9 +225,7 @@ def _is_thought_part(part: dict[str, Any]) -> bool: return bool(metadata.get("adk_thought")) @classmethod - def _split_text_parts( - cls, parts: list[dict[str, Any]] - ) -> tuple[str, str | None]: + def _split_text_parts(cls, parts: list[dict[str, Any]]) -> tuple[str, str | None]: """Split text parts into visible message text and reasoning content.""" message_parts: list[str] = [] reasoning_parts: list[str] = [] @@ -328,9 +319,7 @@ def _collect_response_parts(self, result: dict[str, Any]) -> list[dict[str, Any] return parts - def _build_trajectory( - self, result: dict[str, Any], instruction: str - ) -> dict[str, Any]: + def _build_trajectory(self, result: dict[str, Any], instruction: str) -> dict[str, Any]: """Build an ATIF v1.7 trajectory from the A2A task result.""" session_id = result.get("id") or result.get("contextId") or str(uuid.uuid4()) @@ -356,9 +345,7 @@ def _build_trajectory( ) step_id += 1 elif role == "agent": - steps.append( - self._history_message_to_agent_step(parts, step_id) - ) + steps.append(self._history_message_to_agent_step(parts, step_id)) step_id += 1 else: steps.append( @@ -372,9 +359,7 @@ def _build_trajectory( response_parts = self._collect_response_parts(result) if response_parts: - steps.append( - self._history_message_to_agent_step(response_parts, step_id) - ) + steps.append(self._history_message_to_agent_step(response_parts, step_id)) else: steps.append( { @@ -388,12 +373,8 @@ def _build_trajectory( final_metrics: dict[str, Any] = {"total_steps": len(steps)} if usage: final_metrics["total_prompt_tokens"] = usage.get("promptTokenCount") - final_metrics["total_completion_tokens"] = usage.get( - "candidatesTokenCount" - ) - final_metrics["total_cached_tokens"] = usage.get( - "cachedContentTokenCount" - ) + final_metrics["total_completion_tokens"] = usage.get("candidatesTokenCount") + final_metrics["total_cached_tokens"] = usage.get("cachedContentTokenCount") agent_info: dict[str, Any] = { "name": self.name(), @@ -497,14 +478,10 @@ async def _write_trajectory_file( ) if EnvironmentPaths is None: - self.logger.warning( - "Harbor not available, skipping trajectory container upload" - ) + self.logger.warning("Harbor not available, skipping trajectory container upload") return - trajectory_path_agent = str( - EnvironmentPaths.agent_dir / A2A_TRAJECTORY_FILE - ) + trajectory_path_agent = str(EnvironmentPaths.agent_dir / A2A_TRAJECTORY_FILE) if environment.capabilities.mounted: return diff --git a/abevalflow/harbor_agents/verifiers/llm_judge.py b/abevalflow/harbor_agents/verifiers/llm_judge.py index 7e35009..f709f08 100644 --- a/abevalflow/harbor_agents/verifiers/llm_judge.py +++ b/abevalflow/harbor_agents/verifiers/llm_judge.py @@ -31,9 +31,7 @@ def grade(response_text: str) -> dict: DEFAULT_CRITERIA = ["correctness", "helpfulness", "safety"] DEFAULT_MODEL = os.environ.get("LLM_JUDGE_MODEL", "openai/claude-sonnet") -DEFAULT_LLM_BASE_URL = os.environ.get( - "LLM_BASE_URL", "http://litellm.ab-eval-flow.svc.cluster.local:4000" -) +DEFAULT_LLM_BASE_URL = os.environ.get("LLM_BASE_URL", "http://litellm.ab-eval-flow.svc.cluster.local:4000") JUDGE_SYSTEM_PROMPT = """You are an expert evaluator for AI agent responses. Your task is to score the agent's response on the following criteria: @@ -172,8 +170,7 @@ def _call_judge(self, response_text: str) -> JudgeResult: return self._fallback_score(response_text) criteria_desc = "\n".join( - f"- {name}: {CRITERIA_DESCRIPTIONS.get(name, 'Custom criterion')}" - for name in self.criteria + f"- {name}: {CRITERIA_DESCRIPTIONS.get(name, 'Custom criterion')}" for name in self.criteria ) system_prompt = JUDGE_SYSTEM_PROMPT.format(criteria_descriptions=criteria_desc) @@ -227,9 +224,7 @@ def _build_user_prompt(self, response_text: str) -> str: parts.append(f"## Agent Response\n{response_text}") - parts.append( - "\nPlease evaluate the agent's response on the specified criteria." - ) + parts.append("\nPlease evaluate the agent's response on the specified criteria.") return "\n\n".join(parts) @@ -257,8 +252,7 @@ def _fallback_score(self, response_text: str) -> JudgeResult: return JudgeResult( overall_score=score, criteria_scores={ - name: {"score": score, "explanation": "Fallback scoring (LLM unavailable)"} - for name in self.criteria + name: {"score": score, "explanation": "Fallback scoring (LLM unavailable)"} for name in self.criteria }, overall_explanation="Fallback scoring due to LLM unavailability", ) diff --git a/abevalflow/llm_client.py b/abevalflow/llm_client.py index 7884add..c6339dc 100644 --- a/abevalflow/llm_client.py +++ b/abevalflow/llm_client.py @@ -31,7 +31,7 @@ def _resolve_config() -> dict: return {"base_url": base_url, "api_key": api_key} -def get_client() -> "OpenAI": +def get_client() -> OpenAI: """Return a configured OpenAI client.""" from openai import OpenAI diff --git a/abevalflow/mcpchecker_report.py b/abevalflow/mcpchecker_report.py index 7f8306b..db0b198 100644 --- a/abevalflow/mcpchecker_report.py +++ b/abevalflow/mcpchecker_report.py @@ -10,7 +10,6 @@ from __future__ import annotations -from datetime import datetime, timezone from typing import Literal from pydantic import BaseModel, Field, computed_field @@ -21,9 +20,7 @@ class LLMJudgeResult(BaseModel): """Result from a single LLM judge verification check.""" - check_type: Literal["contains", "exact"] = Field( - description="Type of LLM judge check: 'contains' or 'exact'" - ) + check_type: Literal["contains", "exact"] = Field(description="Type of LLM judge check: 'contains' or 'exact'") expected: str = Field(description="Expected content or reference answer") passed: bool = Field(description="Whether the check passed") reason: str | None = Field( @@ -46,9 +43,7 @@ class MCPCheckerTaskResult(BaseModel): task_id: str = Field(description="Unique task identifier from metadata.name") task_name: str = Field(description="Human-readable task name") - status: Literal["passed", "failed", "error", "skipped"] = Field( - description="Task execution status" - ) + status: Literal["passed", "failed", "error", "skipped"] = Field(description="Task execution status") tool_calls: int = Field(default=0, description="Number of tool calls made") tool_call_records: list[ToolCallRecord] = Field( default_factory=list, diff --git a/abevalflow/report.py b/abevalflow/report.py index bcadd5f..083f507 100644 --- a/abevalflow/report.py +++ b/abevalflow/report.py @@ -7,7 +7,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from enum import StrEnum from pydantic import BaseModel, Field, computed_field @@ -111,7 +111,7 @@ class VariantSummary(BaseModel): class Provenance(BaseModel): """Run provenance metadata for reproducibility.""" - generated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + generated_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) commit_sha: str | None = None pipeline_run_id: str | None = None treatment_image_ref: str | None = None diff --git a/abevalflow/schemas.py b/abevalflow/schemas.py index 38e3430..ad9d05a 100644 --- a/abevalflow/schemas.py +++ b/abevalflow/schemas.py @@ -275,8 +275,7 @@ def _validate_thresholds(cls, v: dict[str, float] | None) -> dict[str, float] | for check_id, threshold in v.items(): if check_id not in valid_check_ids: raise ValueError( - f"Invalid threshold key '{check_id}'. " - f"Must be a valid CheckId. Valid IDs: {sorted(valid_check_ids)}" + f"Invalid threshold key '{check_id}'. Must be a valid CheckId. Valid IDs: {sorted(valid_check_ids)}" ) if not 0.0 <= threshold <= 1.0: raise ValueError(f"Threshold for {check_id} must be between 0.0 and 1.0") @@ -404,9 +403,7 @@ class ExperimentConfig(BaseModel): default=ExperimentType.SKILL, description="Experiment type: skill, model, prompt, custom", ) - n_trials: int = Field( - default=20, gt=0, le=100, description="Number of trials per variant" - ) + n_trials: int = Field(default=20, gt=0, le=100, description="Number of trials per variant") treatment: VariantSpec = Field( default_factory=lambda: VariantSpec( copy=[ @@ -445,7 +442,9 @@ def _validate_schema_version(cls, v: str) -> str: raise ValueError("schema_version must be in 'MAJOR.MINOR' format (e.g. '1.0')") return v - name: str = Field(min_length=1, description="Submission name (OCI-safe: lowercase, alphanumeric, hyphens, dots, underscores)") + name: str = Field( + min_length=1, description="Submission name (OCI-safe: lowercase, alphanumeric, hyphens, dots, underscores)" + ) @field_validator("name") @classmethod @@ -475,16 +474,14 @@ def _validate_name(cls, v: str) -> str: skip_llm_judge: bool = Field( default=False, description=( - "Set to true to skip LLM judge generation in AI mode. " - "By default the pipeline generates tests/llm_judge.py." + "Set to true to skip LLM judge generation in AI mode. By default the pipeline generates tests/llm_judge.py." ), ) experiment: ExperimentConfig = Field( default_factory=ExperimentConfig, description=( - "A/B experiment configuration. Omit entirely to use the default " - "skill experiment with N=20 trials." + "A/B experiment configuration. Omit entirely to use the default skill experiment with N=20 trials." ), ) diff --git a/abevalflow/scorecard.py b/abevalflow/scorecard.py index 3488988..45ff8e9 100644 --- a/abevalflow/scorecard.py +++ b/abevalflow/scorecard.py @@ -7,7 +7,7 @@ based on which checks pass. See abevalflow.certification for level definitions. """ -from datetime import datetime, timezone +from datetime import UTC, datetime from enum import StrEnum from typing import Any @@ -16,7 +16,6 @@ from abevalflow.certification import ( CertificationLevel, CertificationResult, - compute_certification, ) from abevalflow.compass_facts import CertificationFactPushResult, FactPushResult from abevalflow.gates.base import GateMode, GateResult @@ -52,15 +51,13 @@ class Scorecard(BaseModel): description="Policy that was applied for this evaluation", ) - recommendation: Recommendation = Field( - description="Unified verdict: pass, warn, or fail" - ) + recommendation: Recommendation = Field(description="Unified verdict: pass, warn, or fail") recommendation_reason: str = Field( description="Human-readable explanation of the recommendation", ) created_at: datetime = Field( - default_factory=lambda: datetime.now(timezone.utc), + default_factory=lambda: datetime.now(UTC), description="Timestamp when the scorecard was created", ) diff --git a/abevalflow/skill_loader.py b/abevalflow/skill_loader.py index 27f6ff1..13fb2e1 100644 --- a/abevalflow/skill_loader.py +++ b/abevalflow/skill_loader.py @@ -50,8 +50,14 @@ def fetch_skill( try: subprocess.run( [ - "git", "clone", "--depth", "1", "--sparse", - "--filter=blob:none", repo_url, tmp, + "git", + "clone", + "--depth", + "1", + "--sparse", + "--filter=blob:none", + repo_url, + tmp, ], capture_output=True, text=True, diff --git a/pyproject.toml b/pyproject.toml index 8c9197e..5ea1245 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,10 +34,11 @@ pythonpath = ["."] [tool.ruff] target-version = "py311" -line-length = 100 +line-length = 120 [tool.ruff.lint] select = ["E", "F", "I", "UP", "W"] +ignore = ["E402"] # Allow imports after registry setup in __init__.py [tool.hatch.build.targets.wheel] packages = ["abevalflow"] diff --git a/scripts/aggregate_ase.py b/scripts/aggregate_ase.py index c47cfe3..69e35e7 100644 --- a/scripts/aggregate_ase.py +++ b/scripts/aggregate_ase.py @@ -71,9 +71,7 @@ def _parse_benchmarks(results_dir: Path, n_iterations: int) -> list[dict]: return benchmarks -def _collect_trials( - results_dir: Path, n_iterations: int -) -> tuple[list[TrialResult], list[TrialResult]]: +def _collect_trials(results_dir: Path, n_iterations: int) -> tuple[list[TrialResult], list[TrialResult]]: """Collect per-iteration trial results for treatment (with_skill) and control (without_skill).""" treatment_trials: list[TrialResult] = [] control_trials: list[TrialResult] = [] @@ -193,9 +191,7 @@ def build_ase_analysis( logger.warning("Zero passes in both variants — defaulting to FAIL") recommendation = Recommendation.FAIL else: - recommendation = ( - Recommendation.PASS if primary_gap >= threshold else Recommendation.FAIL - ) + recommendation = Recommendation.PASS if primary_gap >= threshold else Recommendation.FAIL prov = provenance or Provenance() prov.eval_engine = "ase" @@ -262,12 +258,8 @@ def render_markdown(result: AnalysisResult) -> str: lines.append(f"- **Uplift (pass rate gap):** {s.uplift:+.4f}") if s.mean_reward_gap is not None: lines.append(f"- **Mean reward gap:** {s.mean_reward_gap:+.4f}") - lines.append( - f"- **Welch's t-test p-value:** {_fmt(s.ttest_p_value)}{_sig_marker(s.ttest_p_value)}" - ) - lines.append( - f"- **Fisher's exact p-value:** {_fmt(s.fisher_p_value)}{_sig_marker(s.fisher_p_value)}" - ) + lines.append(f"- **Welch's t-test p-value:** {_fmt(s.ttest_p_value)}{_sig_marker(s.ttest_p_value)}") + lines.append(f"- **Fisher's exact p-value:** {_fmt(s.fisher_p_value)}{_sig_marker(s.fisher_p_value)}") lines.append(f"- **Recommendation:** **{s.recommendation.value.upper()}**") lines.append(f"- **Evaluation engine:** {prov.eval_engine}") lines.append("") @@ -303,23 +295,32 @@ def main(argv: list[str] | None = None) -> int: description="Aggregate agent-skills-eval results into AnalysisResult format", ) parser.add_argument( - "--results-dir", type=Path, required=True, + "--results-dir", + type=Path, + required=True, help="Path to ASE results directory containing iteration-N/ subdirs", ) parser.add_argument( - "--output-dir", type=Path, required=True, + "--output-dir", + type=Path, + required=True, help="Directory to write report.json and report.md", ) parser.add_argument( - "--submission-name", required=True, + "--submission-name", + required=True, help="Name of the submission being analyzed", ) parser.add_argument( - "--iterations", type=int, default=5, + "--iterations", + type=int, + default=5, help="Number of iterations to aggregate (default: 5)", ) parser.add_argument( - "--threshold", type=float, default=0.0, + "--threshold", + type=float, + default=0.0, help="Minimum uplift for a 'pass' recommendation (default: 0.0)", ) parser.add_argument("--commit-sha", default=None) diff --git a/scripts/aggregate_mcpchecker.py b/scripts/aggregate_mcpchecker.py index 05ea9e6..eb3f8f5 100644 --- a/scripts/aggregate_mcpchecker.py +++ b/scripts/aggregate_mcpchecker.py @@ -18,7 +18,7 @@ import json import logging import sys -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from abevalflow.mcpchecker_report import ( @@ -47,7 +47,7 @@ def parse_mcpchecker_output(output_path: Path) -> dict: def extract_task_results(raw_output: dict) -> list[MCPCheckerTaskResult]: """Extract task results from MCPChecker output format. - + Handles multiple MCPChecker output formats: - v1alpha2 format: results[] with taskName, taskPassed, callHistory, assertionResults - Legacy format: taskResults[] with taskId, status, toolCalls, llmJudgeResults @@ -90,15 +90,15 @@ def extract_task_results(raw_output: dict) -> list[MCPCheckerTaskResult]: # Fallback to legacy toolCalls field if not tool_calls_data: tool_calls_data = task_data.get("toolCalls") or [] - + tool_call_records = [] - for tc in (tool_calls_data or []): + for tc in tool_calls_data or []: if isinstance(tc, dict): # Extract arguments from request.Params.arguments if present request = tc.get("request") or {} params = request.get("Params") or {} arguments = params.get("arguments") or tc.get("arguments") or tc.get("params") - + tool_call_records.append( ToolCallRecord( server=tc.get("serverName", tc.get("server", tc.get("mcpServer", "unknown"))), @@ -110,14 +110,14 @@ def extract_task_results(raw_output: dict) -> list[MCPCheckerTaskResult]: # Handle judge/assertion results (v1alpha2 uses assertionResults, legacy uses llmJudgeResults) llm_judge_data = ( - task_data.get("llmJudgeResults") or - task_data.get("verifyResults") or - task_data.get("assertionResults") or - [] + task_data.get("llmJudgeResults") + or task_data.get("verifyResults") + or task_data.get("assertionResults") + or [] ) llm_judge_results = [] - for judge in (llm_judge_data or []): + for judge in llm_judge_data or []: if isinstance(judge, dict): llm_judge_results.append( LLMJudgeResult( @@ -181,7 +181,7 @@ def aggregate_mcpchecker_results( total_duration_ms = sum(task_durations) provenance = Provenance( - generated_at=datetime.now(timezone.utc), + generated_at=datetime.now(UTC), commit_sha=commit_sha, pipeline_run_id=pipeline_run_id, eval_engine="mcpchecker", @@ -204,9 +204,7 @@ def aggregate_mcpchecker_results( def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description="Aggregate MCPChecker output into report format" - ) + parser = argparse.ArgumentParser(description="Aggregate MCPChecker output into report format") parser.add_argument( "--output-json", type=Path, @@ -254,7 +252,7 @@ def main(argv: list[str] | None = None) -> int: with open(report_path, "w") as f: f.write(result.model_dump_json(indent=2)) - + # Also write to report.json for compatibility with analyze task with open(compat_report_path, "w") as f: f.write(result.model_dump_json(indent=2)) @@ -268,13 +266,17 @@ def main(argv: list[str] | None = None) -> int: result.recommendation, ) - print(json.dumps({ - "report_path": str(report_path), - "overall_score": result.overall_score, - "passed_tasks": result.passed_tasks, - "total_tasks": result.total_tasks, - "recommendation": result.recommendation, - })) + print( + json.dumps( + { + "report_path": str(report_path), + "overall_score": result.overall_score, + "passed_tasks": result.passed_tasks, + "total_tasks": result.total_tasks, + "recommendation": result.recommendation, + } + ) + ) return 0 diff --git a/scripts/aggregate_scorecard.py b/scripts/aggregate_scorecard.py index 4a71f39..b5e4c7c 100644 --- a/scripts/aggregate_scorecard.py +++ b/scripts/aggregate_scorecard.py @@ -27,7 +27,7 @@ import json import logging import sys -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path import yaml @@ -46,7 +46,7 @@ from abevalflow.gates.quality import get_all_quality_gates from abevalflow.gates.security import get_all_security_gates from abevalflow.schemas import CertificationPolicy, GatePolicy, SubmissionMetadata -from abevalflow.scorecard import Recommendation, Scorecard, apply_combination_logic +from abevalflow.scorecard import Scorecard, apply_combination_logic logging.basicConfig( level=logging.INFO, @@ -99,7 +99,7 @@ def load_certification_policy( CertificationPolicy or None for defaults """ metadata_path = submission_dir / "metadata.yaml" - + # Try loading from metadata.yaml first if metadata_path.exists(): try: @@ -136,9 +136,7 @@ def load_provenance(reports_dir: Path) -> dict: return {} -def _maybe_push_fact( - gate_result: GateResult, policy: GatePolicy -) -> FactPushResult | None: +def _maybe_push_fact(gate_result: GateResult, policy: GatePolicy) -> FactPushResult | None: """Push gate fact to Compass if configured. Args: @@ -246,10 +244,7 @@ def aggregate_scorecard( push_result = _maybe_push_fact(engine_gate, policy) if push_result: fact_push_results.append(push_result) - logger.info( - "Engine %s: passed=%s, score=%.3f", - engine.name, engine_gate.passed, engine_gate.score - ) + logger.info("Engine %s: passed=%s, score=%.3f", engine.name, engine_gate.passed, engine_gate.score) else: logger.warning("No result found for engine %s at %s", engine.name, engine_reports_dir) @@ -266,7 +261,10 @@ def aggregate_scorecard( fact_push_results.append(push_result) logger.info( "Security %s: passed=%s, score=%.3f, findings=%d", - security_gate.name, gate_result.passed, gate_result.score, len(gate_result.findings) + security_gate.name, + gate_result.passed, + gate_result.score, + len(gate_result.findings), ) for quality_gate in get_all_quality_gates(): @@ -280,10 +278,7 @@ def aggregate_scorecard( push_result = _maybe_push_fact(gate_result, policy) if push_result: fact_push_results.append(push_result) - logger.info( - "Quality %s: passed=%s, score=%.3f", - quality_gate.name, gate_result.passed, gate_result.score - ) + logger.info("Quality %s: passed=%s, score=%.3f", quality_gate.name, gate_result.passed, gate_result.score) recommendation, reason = apply_combination_logic(gates, policy) logger.info("Final recommendation: %s (%s)", recommendation, reason) @@ -318,10 +313,7 @@ def aggregate_scorecard( validation_passed = False metadata_valid = False - has_eval_assets = ( - (submission_dir / "evals" / "evals.json").exists() - or (submission_dir / "tests").exists() - ) + has_eval_assets = (submission_dir / "evals" / "evals.json").exists() or (submission_dir / "tests").exists() certification = compute_certification( gates=gates, @@ -367,7 +359,7 @@ def aggregate_scorecard( policy=policy, recommendation=recommendation, recommendation_reason=reason, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), provenance=provenance, fact_push_results=fact_push_results, certification=certification, @@ -388,9 +380,7 @@ def write_scorecard(scorecard: Scorecard, reports_dir: Path) -> Path: def main() -> int: - parser = argparse.ArgumentParser( - description="Aggregate gate results into unified scorecard" - ) + parser = argparse.ArgumentParser(description="Aggregate gate results into unified scorecard") parser.add_argument( "--submission-dir", type=Path, @@ -439,7 +429,7 @@ def main() -> int: type=str, default=None, help="Certification profile name (e.g., 'skill', 'agent', 'mcp_server'). " - "Provides default checks per artifact type. Overridden by submission's metadata.yaml.", + "Provides default checks per artifact type. Overridden by submission's metadata.yaml.", ) args = parser.parse_args() diff --git a/scripts/alert.py b/scripts/alert.py index 6964e4c..9b43b5e 100644 --- a/scripts/alert.py +++ b/scripts/alert.py @@ -15,10 +15,9 @@ import json import logging import sys -from typing import Any - -import urllib.request import urllib.error +import urllib.request +from typing import Any logging.basicConfig( level=logging.INFO, @@ -62,9 +61,7 @@ def format_slack_message( header_text = f"✅ [{engine_label}] Monitoring Pass" run_id_text = ( - f"<{pipeline_run_url}|{current_run_id}>" - if pipeline_run_url and current_run_id != "N/A" - else current_run_id + f"<{pipeline_run_url}|{current_run_id}>" if pipeline_run_url and current_run_id != "N/A" else current_run_id ) fields = [ diff --git a/scripts/analyze.py b/scripts/analyze.py index 8cfd3a5..8ac4f61 100644 --- a/scripts/analyze.py +++ b/scripts/analyze.py @@ -60,6 +60,7 @@ # Security scan parsing # --------------------------------------------------------------------------- + def parse_security_scan(report_dir: Path, scan_mode: str | None) -> SecurityScanResult | None: """Parse security-scan.json if it exists in the report directory. @@ -83,15 +84,19 @@ def parse_security_scan(report_dir: Path, scan_mode: str | None) -> SecurityScan for f in data.get("findings", []): try: sev_str = f.get("severity", "info").lower() - severity = SecuritySeverity(sev_str) if sev_str in SecuritySeverity.__members__.values() else SecuritySeverity.INFO - findings.append(SecurityFinding( - rule_id=f.get("rule_id", f.get("id", "unknown")), - severity=severity, - message=f.get("message", f.get("description", "")), - file_path=f.get("file_path", f.get("location", {}).get("file")), - line_number=f.get("line_number", f.get("location", {}).get("line")), - scanner="cisco", - )) + severity = ( + SecuritySeverity(sev_str) if sev_str in SecuritySeverity.__members__.values() else SecuritySeverity.INFO + ) + findings.append( + SecurityFinding( + rule_id=f.get("rule_id", f.get("id", "unknown")), + severity=severity, + message=f.get("message", f.get("description", "")), + file_path=f.get("file_path", f.get("location", {}).get("file")), + line_number=f.get("line_number", f.get("location", {}).get("line")), + scanner="cisco", + ) + ) except Exception as e: logger.warning("Failed to parse finding: %s", e) @@ -110,6 +115,7 @@ def parse_security_scan(report_dir: Path, scan_mode: str | None) -> SecurityScan # Result parsing # --------------------------------------------------------------------------- + def _extract_reward(result: dict) -> float | None: """Extract reward from a Harbor result.json. @@ -160,10 +166,7 @@ def is_a2a_results(results_dir: Path) -> bool: a2a_dir = results_dir / "a2a-eval" if not a2a_dir.is_dir(): return False - has_ab_variants = ( - (results_dir / "treatment").is_dir() - or (results_dir / "control").is_dir() - ) + has_ab_variants = (results_dir / "treatment").is_dir() or (results_dir / "control").is_dir() return not has_ab_variants @@ -189,7 +192,8 @@ def build_a2a_analysis( if 0 < summary.n_trials < _MIN_TRIALS_FOR_RELIABLE_STATS: logger.warning( "A2A has only %d trials (< %d) — statistics may be unreliable", - summary.n_trials, _MIN_TRIALS_FOR_RELIABLE_STATS, + summary.n_trials, + _MIN_TRIALS_FOR_RELIABLE_STATS, ) if summary.n_trials == 0: @@ -201,9 +205,7 @@ def build_a2a_analysis( else: pass_threshold = threshold if threshold > 0 else 0.5 mean_r = summary.mean_reward or 0.0 - recommendation = ( - Recommendation.PASS if mean_r >= pass_threshold else Recommendation.FAIL - ) + recommendation = Recommendation.PASS if mean_r >= pass_threshold else Recommendation.FAIL return AnalysisResult( submission_name=submission_name, @@ -227,6 +229,7 @@ def build_a2a_analysis( # Statistics # --------------------------------------------------------------------------- + def compute_variant_summary(trials: list[TrialResult]) -> VariantSummary: """Compute aggregate stats from a list of trial results.""" rewards = [t.reward for t in trials if t.reward is not None] @@ -252,8 +255,7 @@ def compute_variant_summary(trials: list[TrialResult]) -> VariantSummary: ) -def compute_ttest(treatment_trials: list[TrialResult], - control_trials: list[TrialResult]) -> float | None: +def compute_ttest(treatment_trials: list[TrialResult], control_trials: list[TrialResult]) -> float | None: """Welch's t-test on continuous reward scores between variants.""" t_rewards = [t.reward for t in treatment_trials if t.reward is not None] c_rewards = [t.reward for t in control_trials if t.reward is not None] @@ -265,8 +267,7 @@ def compute_ttest(treatment_trials: list[TrialResult], return float(p) -def compute_fisher(treatment_summary: VariantSummary, - control_summary: VariantSummary) -> float | None: +def compute_fisher(treatment_summary: VariantSummary, control_summary: VariantSummary) -> float | None: """Fisher's exact test on the 2x2 pass/fail contingency table. Error trials (missing/corrupt results) are excluded from the table so @@ -287,6 +288,7 @@ def compute_fisher(treatment_summary: VariantSummary, # Degradation merge # --------------------------------------------------------------------------- + def degradation_from_monitor(monitor_data: dict) -> DegradationResult: """Build a DegradationResult from monitor.py JSON output.""" return DegradationResult( @@ -320,6 +322,7 @@ def merge_degradation_into_report( # Report generation # --------------------------------------------------------------------------- + def build_analysis( results_dir: Path, submission_name: str, @@ -351,7 +354,9 @@ def build_analysis( if 0 < vs.n_trials < _MIN_TRIALS_FOR_RELIABLE_STATS: logger.warning( "%s has only %d trials (< %d) — statistical tests may be unreliable", - label, vs.n_trials, _MIN_TRIALS_FOR_RELIABLE_STATS, + label, + vs.n_trials, + _MIN_TRIALS_FOR_RELIABLE_STATS, ) uplift = t_summary.pass_rate - c_summary.pass_rate @@ -467,12 +472,8 @@ def render_markdown(result: AnalysisResult) -> str: lines.append(f"- **Mean reward gap (Uplift):** {s.mean_reward_gap:+.4f}") else: lines.append(f"- **Uplift (pass rate gap):** {s.uplift:+.4f}") - lines.append( - f"- **Welch's t-test p-value:** {_fmt(s.ttest_p_value)}{_sig_marker(s.ttest_p_value)}" - ) - lines.append( - f"- **Fisher's exact p-value:** {_fmt(s.fisher_p_value)}{_sig_marker(s.fisher_p_value)}" - ) + lines.append(f"- **Welch's t-test p-value:** {_fmt(s.ttest_p_value)}{_sig_marker(s.ttest_p_value)}") + lines.append(f"- **Fisher's exact p-value:** {_fmt(s.fisher_p_value)}{_sig_marker(s.fisher_p_value)}") lines.append(f"- **Recommendation:** **{s.recommendation.value.upper()}**") lines.append("") @@ -566,6 +567,7 @@ def render_markdown(result: AnalysisResult) -> str: # CLI # --------------------------------------------------------------------------- + def main(argv: list[str] | None = None) -> int: logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") @@ -573,11 +575,13 @@ def main(argv: list[str] | None = None) -> int: description="Analyze A/B evaluation results and produce JSON + Markdown reports", ) parser.add_argument( - "--results-dir", type=Path, + "--results-dir", + type=Path, help="Path to the results directory containing treatment/ and control/ subdirs", ) parser.add_argument( - "--output-dir", type=Path, + "--output-dir", + type=Path, help="Directory to write report.json and report.md", ) parser.add_argument( @@ -585,15 +589,21 @@ def main(argv: list[str] | None = None) -> int: help="Name of the submission being analyzed", ) parser.add_argument( - "--merge-degradation-from", type=Path, default=None, + "--merge-degradation-from", + type=Path, + default=None, help="Path to monitor.py JSON output to merge into an existing report.json", ) parser.add_argument( - "--report-json", type=Path, default=None, + "--report-json", + type=Path, + default=None, help="Path to report.json when merging degradation results", ) parser.add_argument( - "--threshold", type=float, default=0.0, + "--threshold", + type=float, + default=0.0, help="Minimum uplift for a 'pass' recommendation (default: 0.0)", ) parser.add_argument("--commit-sha", default=None) @@ -635,10 +645,7 @@ def main(argv: list[str] | None = None) -> int: if report_path is None and args.output_dir is not None: report_path = args.output_dir / "report.json" if report_path is None or not report_path.is_file(): - logger.error( - "report.json not found for degradation merge " - "(use --report-json or --output-dir)" - ) + logger.error("report.json not found for degradation merge (use --report-json or --output-dir)") return 1 if not args.merge_degradation_from.is_file(): logger.error("Monitor output file does not exist: %s", args.merge_degradation_from) @@ -647,9 +654,7 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.results_dir is None or args.output_dir is None or args.submission_name is None: - logger.error( - "--results-dir, --output-dir, and --submission-name are required for analysis" - ) + logger.error("--results-dir, --output-dir, and --submission-name are required for analysis") return 1 if not args.results_dir.is_dir(): diff --git a/scripts/generate_ase_evals.py b/scripts/generate_ase_evals.py index e68a66d..c2b040f 100644 --- a/scripts/generate_ase_evals.py +++ b/scripts/generate_ase_evals.py @@ -24,16 +24,14 @@ from pathlib import Path # Import from generate_tests to reuse the existing infrastructure -from scripts.generate_tests import generate_ase_evals, _load_submission +from scripts.generate_tests import _load_submission, generate_ase_evals logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger(__name__) def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description="Generate evals/evals.json from SKILL.md for ASE evaluation" - ) + parser = argparse.ArgumentParser(description="Generate evals/evals.json from SKILL.md for ASE evaluation") parser.add_argument( "submission_dir", type=Path, diff --git a/scripts/generate_eval_config.py b/scripts/generate_eval_config.py index 694d396..830471c 100644 --- a/scripts/generate_eval_config.py +++ b/scripts/generate_eval_config.py @@ -109,12 +109,14 @@ def build_variant_config( directory and trial classification is unambiguous. """ if eval_mode == "prebuilt" and not image_ref: - raise ValueError( - f"image_ref is required for variant '{variant}' in prebuilt mode" - ) + raise ValueError(f"image_ref is required for variant '{variant}' in prebuilt mode") llm_model, llm_api_base, llm_api_key, llm_agent_wrapper = _resolve_llm_params( - metadata, llm_model, llm_api_base, llm_api_key, llm_agent_wrapper, + metadata, + llm_model, + llm_api_base, + llm_api_key, + llm_agent_wrapper, ) task: dict[str, Any] = {"path": task_dir} @@ -138,18 +140,10 @@ def build_variant_config( if eval_mode != "prebuilt": env_block["force_build"] = True - agent_mult = _timeout_multiplier( - metadata.agent_timeout_sec, _HARBOR_DEFAULT_AGENT_TIMEOUT - ) - verifier_mult = _timeout_multiplier( - metadata.verifier_timeout_sec, _HARBOR_DEFAULT_VERIFIER_TIMEOUT - ) - setup_mult = _timeout_multiplier( - metadata.agent_setup_timeout_sec, _HARBOR_DEFAULT_SETUP_TIMEOUT - ) - build_mult = _timeout_multiplier( - metadata.build_timeout_sec, _HARBOR_DEFAULT_BUILD_TIMEOUT - ) + agent_mult = _timeout_multiplier(metadata.agent_timeout_sec, _HARBOR_DEFAULT_AGENT_TIMEOUT) + verifier_mult = _timeout_multiplier(metadata.verifier_timeout_sec, _HARBOR_DEFAULT_VERIFIER_TIMEOUT) + setup_mult = _timeout_multiplier(metadata.agent_setup_timeout_sec, _HARBOR_DEFAULT_SETUP_TIMEOUT) + build_mult = _timeout_multiplier(metadata.build_timeout_sec, _HARBOR_DEFAULT_BUILD_TIMEOUT) agent_config: dict[str, Any] = {} if llm_agent_wrapper: @@ -205,11 +199,12 @@ def generate_eval_configs( metadata = load_metadata(submission_dir) output_dir.mkdir(parents=True, exist_ok=True) - variant_args = dict(zip( - VARIANTS, - ((treatment_task_dir, treatment_image_ref), - (control_task_dir, control_image_ref)), - )) + variant_args = dict( + zip( + VARIANTS, + ((treatment_task_dir, treatment_image_ref), (control_task_dir, control_image_ref)), + ) + ) configs: dict[str, dict[str, Any]] = {} for variant, (task_dir, img_ref) in variant_args.items(): @@ -309,10 +304,7 @@ def main(argv: list[str] | None = None) -> int: if args.eval_mode == "prebuilt": if not args.treatment_image_ref or not args.control_image_ref: - parser.error( - "--treatment-image-ref and --control-image-ref are required " - "when --eval-mode is 'prebuilt'" - ) + parser.error("--treatment-image-ref and --control-image-ref are required when --eval-mode is 'prebuilt'") if not args.submission_dir.is_dir(): logger.error("Submission directory does not exist: %s", args.submission_dir) diff --git a/scripts/generate_tests.py b/scripts/generate_tests.py index bf0b6c0..fadad90 100644 --- a/scripts/generate_tests.py +++ b/scripts/generate_tests.py @@ -39,7 +39,6 @@ ase_evals_review, check_markdown, check_python, - content_check, final_review, multi_reviewer_check, pytest_collect_check, @@ -47,6 +46,7 @@ structural_check, ) from abevalflow.schemas import SubmissionMetadata + logger = logging.getLogger(__name__) DEFAULT_MAX_RETRIES = 5 @@ -483,10 +483,7 @@ def _error_block(errors: list[str] | None) -> str: if not errors: return "" lines = "\n".join(f"- {e}" for e in errors) - return ( - f"\n## Previous Attempt Issues\n" - f"Your previous attempt had these issues — fix them:\n{lines}\n" - ) + return f"\n## Previous Attempt Issues\nYour previous attempt had these issues — fix them:\n{lines}\n" def _llm_call(system: str, user: str, *, max_tokens: int = 4096) -> str: @@ -502,7 +499,7 @@ def _llm_call(system: str, user: str, *, max_tokens: int = 4096) -> str: text = raw.strip() for fence in ("```python", "```markdown", "```md", "```"): if text.startswith(fence): - text = text[len(fence):] + text = text[len(fence) :] break if text.endswith("```"): text = text[:-3] @@ -598,11 +595,14 @@ def generate_ase_evals( raw = llm_client.chat_completion( messages=[ {"role": "system", "content": ASE_EVALS_SYSTEM_PROMPT}, - {"role": "user", "content": ASE_EVALS_PROMPT.format( - skill_name=skill_name, - skill_content=skill_content, - skill_analysis=analysis_text, - )}, + { + "role": "user", + "content": ASE_EVALS_PROMPT.format( + skill_name=skill_name, + skill_content=skill_content, + skill_analysis=analysis_text, + ), + }, ], temperature=0.3, max_tokens=4096, @@ -612,7 +612,7 @@ def generate_ase_evals( text = raw.strip() for fence in ("```json", "```"): if text.startswith(fence): - text = text[len(fence):] + text = text[len(fence) :] break if text.endswith("```"): text = text[:-3] @@ -749,9 +749,7 @@ def _generate_via_api( """ system = SYSTEM_PROMPT.format( quality_criteria=( - f"\n## Quality Criteria (from agentic-contribution-skill)\n\n{quality_criteria}" - if quality_criteria - else "" + f"\n## Quality Criteria (from agentic-contribution-skill)\n\n{quality_criteria}" if quality_criteria else "" ), ) @@ -765,7 +763,9 @@ def _generate_via_api( # --- Step 0.5: scenario brief --------------------------------------------- brief, brief_json = _generate_scenario_brief( - skill_content, analysis_text, system, + skill_content, + analysis_text, + system, ) (submission_dir / "scenario_brief.json").write_text(brief_json) @@ -806,9 +806,7 @@ def _generate_via_api( ), max_tokens=65536, ) - (tests_dir / "test_outputs.py").write_text( - ("# Generated by AI\n" + test_text) if test_text.strip() else test_text - ) + (tests_dir / "test_outputs.py").write_text(("# Generated by AI\n" + test_text) if test_text.strip() else test_text) logger.info("Step 2: generated test_outputs.py (%d chars)", len(test_text)) errors = check_python(tests_dir / "test_outputs.py") @@ -871,8 +869,7 @@ def _generate_via_agent( if agent_type != "api": logger.warning( - "Agent mode '%s' executes with elevated permissions — " - "ensure the submission source is trusted", + "Agent mode '%s' executes with elevated permissions — ensure the submission source is trusted", agent_type, ) @@ -922,9 +919,7 @@ def _generate_via_oracle( """ oracle_dir = submission_dir / "oracle" if not oracle_dir.is_dir(): - raise ValueError( - f"Oracle mode requires an oracle/ directory in {submission_dir}" - ) + raise ValueError(f"Oracle mode requires an oracle/ directory in {submission_dir}") import shutil @@ -1002,7 +997,7 @@ def _correction_pass( system: str = CORRECTION_SYSTEM_PROMPT, ) -> None: """Apply targeted corrections based on consolidated reviewer feedback.""" - issues_text = "\n".join(f"- {issue}" for issue in issues) + _issues_text = "\n".join(f"- {issue}" for issue in issues) # Reserved for prompt use logger.info("Correction pass: addressing %d issues", len(issues)) instruction_path = submission_dir / "instruction.md" @@ -1106,7 +1101,10 @@ def generate( ) else: generated = _generate_via_agent( - submission_dir, skill_dir, agent_type, workspace_dir, + submission_dir, + skill_dir, + agent_type, + workspace_dir, ) except (ValueError, RuntimeError) as exc: last_errors = [str(exc)] @@ -1120,12 +1118,13 @@ def generate( if struct_errors: last_errors = struct_errors logger.warning( - "Attempt %d structural validation failed: %s", attempt, struct_errors, + "Attempt %d structural validation failed: %s", + attempt, + struct_errors, ) if attempt == max_retries: raise ValueError( - f"Generation failed structural validation after {max_retries} " - f"attempts: {struct_errors}" + f"Generation failed structural validation after {max_retries} attempts: {struct_errors}" ) continue @@ -1134,13 +1133,12 @@ def generate( if collect_errors: last_errors = collect_errors logger.warning( - "Attempt %d pytest collect failed: %s", attempt, collect_errors, + "Attempt %d pytest collect failed: %s", + attempt, + collect_errors, ) if attempt == max_retries: - raise ValueError( - f"Generation failed pytest collect after {max_retries} " - f"attempts: {collect_errors}" - ) + raise ValueError(f"Generation failed pytest collect after {max_retries} attempts: {collect_errors}") continue # --- Step 5.5: scenario coherence check -------------------------------- @@ -1149,12 +1147,13 @@ def generate( if coherence_errors: last_errors = coherence_errors logger.warning( - "Attempt %d coherence check failed: %s", attempt, coherence_errors, + "Attempt %d coherence check failed: %s", + attempt, + coherence_errors, ) if attempt == max_retries: raise ValueError( - f"Generation failed coherence check after {max_retries} " - f"attempts: {coherence_errors}" + f"Generation failed coherence check after {max_retries} attempts: {coherence_errors}" ) continue @@ -1163,7 +1162,8 @@ def generate( if not review["passed"]: logger.info( "Attempt %d: reviewers found %d issues, running correction pass", - attempt, len(review["issues"]), + attempt, + len(review["issues"]), ) # --- Step 7: correction pass --------------------------------------- @@ -1175,12 +1175,12 @@ def generate( last_errors = post_errors logger.warning( "Attempt %d post-correction validation failed: %s", - attempt, post_errors, + attempt, + post_errors, ) if attempt == max_retries: raise ValueError( - f"Generation failed post-correction validation after " - f"{max_retries} attempts: {post_errors}" + f"Generation failed post-correction validation after {max_retries} attempts: {post_errors}" ) continue @@ -1231,7 +1231,10 @@ def main(argv: list[str] | None = None) -> int: try: generated = generate( - args.submission_dir, workspace, args.agent_type, args.max_retries, + args.submission_dir, + workspace, + args.agent_type, + args.max_retries, ) except Exception as exc: logger.error("Generation failed: %s", exc) diff --git a/scripts/monitor.py b/scripts/monitor.py index d77cb2b..3b56e8a 100644 --- a/scripts/monitor.py +++ b/scripts/monitor.py @@ -77,9 +77,7 @@ def check_degradation( min_runs = 1 if external_current_score else 2 with Session(engine) as session: - stmt = select(EvaluationRun).where( - EvaluationRun.submission_name == submission_name - ) + stmt = select(EvaluationRun).where(EvaluationRun.submission_name == submission_name) if eval_engine is not None: stmt = stmt.where(EvaluationRun.eval_engine == eval_engine) stmt = stmt.order_by(EvaluationRun.created_at.desc()).limit(limit) @@ -90,20 +88,13 @@ def check_degradation( submission_name=submission_name, degraded=False, current_score=( - current_score - if current_score is not None - else (runs[0].treatment_pass_rate if runs else None) + current_score if current_score is not None else (runs[0].treatment_pass_rate if runs else None) ), previous_score=None, ratio=None, threshold=threshold, - message=( - f"Not enough data: found {len(runs)} run(s), " - f"need at least {min_runs}" - ), - current_run_id=None if external_current_score else ( - runs[0].pipeline_run_id if runs else None - ), + message=(f"Not enough data: found {len(runs)} run(s), need at least {min_runs}"), + current_run_id=None if external_current_score else (runs[0].pipeline_run_id if runs else None), previous_run_id=None, ) @@ -128,10 +119,7 @@ def check_degradation( f"to {current_score:.2%} (ratio: {ratio:.2f} < {threshold})" ) else: - message = ( - f"No degradation: score is {current_score:.2%} " - f"(ratio: {ratio:.2f} >= {threshold})" - ) + message = f"No degradation: score is {current_score:.2%} (ratio: {ratio:.2f} >= {threshold})" return MonitorResult( submission_name=submission_name, diff --git a/scripts/publish.py b/scripts/publish.py index f3f267a..92cebba 100644 --- a/scripts/publish.py +++ b/scripts/publish.py @@ -23,7 +23,7 @@ import subprocess import sys import urllib.request -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from urllib.parse import urlparse @@ -35,7 +35,7 @@ def _build_artifact_prefix( pipeline_run_id: str, ) -> str: """Build the MinIO object prefix: YYYYMMDD_hhmmss__.""" - datestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + datestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") return f"{datestamp}_{submission_name}_{pipeline_run_id}" @@ -330,9 +330,7 @@ def upload_scaffolded_configs( eval_configs_dir = workspace_root / "_eval-configs" if eval_configs_dir.is_dir(): for fpath in sorted(eval_configs_dir.glob("*.yaml")): - masked = re.sub( - r"(API_KEY:\s*)(\S+)", r"\1***", fpath.read_text() - ) + masked = re.sub(r"(API_KEY:\s*)(\S+)", r"\1***", fpath.read_text()) masked_path = fpath.parent / f".masked_{fpath.name}" masked_path.write_text(masked) object_name = f"{prefix}/scaffolded/{fpath.name}" @@ -375,8 +373,7 @@ def upload_scaffolded_configs( except Exception as exc: logger.warning("Failed to upload generated file %s: %s", filepath, exc) - logger.info("Uploaded %d config/generated files to s3://%s/%s/", - uploaded, bucket, prefix) + logger.info("Uploaded %d config/generated files to s3://%s/%s/", uploaded, bucket, prefix) return uploaded @@ -394,7 +391,8 @@ def promote_to_quay( quay_tag = f"{quay_repo}/{submission_name}:treatment-{commit_sha[:12]}" cmd = [ - "skopeo", "copy", + "skopeo", + "copy", "--src-tls-verify=false", f"docker://{treatment_image_ref}", f"docker://{quay_tag}", @@ -428,7 +426,7 @@ def post_pr_comment( Uses the GitHub REST API via ``urllib`` (no ``gh`` CLI required). ``github_token`` falls back to the ``GITHUB_TOKEN`` environment variable. - + If scorecard.json exists, its recommendation takes precedence over report.json. """ import os @@ -466,11 +464,11 @@ def post_pr_comment( sc_reason = scorecard.get("recommendation_reason", "") gates_passed = scorecard.get("gates_passed", 0) gates_failed = scorecard.get("gates_failed", 0) - + # Scorecard recommendation takes precedence if sc_recommendation: recommendation = sc_recommendation - + # Build gate summary gates = scorecard.get("gates", []) gate_lines = [] @@ -480,7 +478,7 @@ def post_pr_comment( score = gate.get("score", 0.0) mode = gate.get("mode", "warn") gate_lines.append(f"| {name} | {status} | {score:.2f} | {mode} |") - + if gate_lines: gate_table = "\n".join(gate_lines) scorecard_section = ( @@ -563,8 +561,7 @@ def cleanup_images( if key in seen_imagestreams: continue seen_imagestreams.add(key) - cmd = ["oc", "delete", "imagestream", is_name, "-n", namespace, - "--ignore-not-found"] + cmd = ["oc", "delete", "imagestream", is_name, "-n", namespace, "--ignore-not-found"] else: cmd = ["oc", "delete", "istag", name_part, "-n", namespace] @@ -590,35 +587,46 @@ def main() -> int: parser.add_argument("--report-dir", type=Path, required=True) parser.add_argument("--submission-name", type=str, required=True) parser.add_argument("--pipeline-run-id", type=str, required=True) - parser.add_argument("--recommendation", type=str, required=True, - help="'pass' or 'fail' from the analyze step") + parser.add_argument("--recommendation", type=str, required=True, help="'pass' or 'fail' from the analyze step") parser.add_argument("--treatment-image-ref", type=str, default="") parser.add_argument("--control-image-ref", type=str, default="") parser.add_argument("--commit-sha", type=str, default="") - parser.add_argument("--uplift-threshold", type=float, default=-1.0, - help="Minimum uplift for Quay promotion. -1.0 disables promotion.") - parser.add_argument("--quay-repo", type=str, default="", - help="Quay.io repo for image promotion (e.g. quay.io/myorg)") + parser.add_argument( + "--uplift-threshold", + type=float, + default=-1.0, + help="Minimum uplift for Quay promotion. -1.0 disables promotion.", + ) + parser.add_argument( + "--quay-repo", type=str, default="", help="Quay.io repo for image promotion (e.g. quay.io/myorg)" + ) parser.add_argument("--quay-ttl-days", type=int, default=7) - parser.add_argument("--repo-name", type=str, default="", - help="GitHub repo (org/name) for PR comment") - parser.add_argument("--pr-number", type=str, default="", - help="PR number for GitHub comment") - parser.add_argument("--results-dir", type=Path, default=None, - help="Path to results dir (Harbor or ASE) for debug artifact upload") - parser.add_argument("--eval-engine", type=str, default="harbor", - choices=["harbor", "ase", "mcpchecker", "a2a", "both"], - help="Evaluation engine used — determines debug artifact layout") - parser.add_argument("--workspace-root", type=Path, default=None, - help="Workspace root for uploading scaffolded configs") - parser.add_argument("--minio-endpoint", type=str, default=None, - help="MinIO endpoint (default: MINIO_ENDPOINT env var)") + parser.add_argument("--repo-name", type=str, default="", help="GitHub repo (org/name) for PR comment") + parser.add_argument("--pr-number", type=str, default="", help="PR number for GitHub comment") + parser.add_argument( + "--results-dir", type=Path, default=None, help="Path to results dir (Harbor or ASE) for debug artifact upload" + ) + parser.add_argument( + "--eval-engine", + type=str, + default="harbor", + choices=["harbor", "ase", "mcpchecker", "a2a", "both"], + help="Evaluation engine used — determines debug artifact layout", + ) + parser.add_argument( + "--workspace-root", type=Path, default=None, help="Workspace root for uploading scaffolded configs" + ) + parser.add_argument( + "--minio-endpoint", type=str, default=None, help="MinIO endpoint (default: MINIO_ENDPOINT env var)" + ) parser.add_argument("--minio-bucket", type=str, default="ab-eval-reports") - parser.add_argument("--report-prefix", type=str, default="", - help="Pre-computed MinIO prefix (skips generating new timestamp)") + parser.add_argument( + "--report-prefix", type=str, default="", help="Pre-computed MinIO prefix (skips generating new timestamp)" + ) args = parser.parse_args() import os + minio_endpoint = args.minio_endpoint or os.environ.get("MINIO_ENDPOINT", "") minio_access_key = os.environ.get("MINIO_ACCESS_KEY", "") minio_secret_key = os.environ.get("MINIO_SECRET_KEY", "") @@ -643,7 +651,8 @@ def main() -> int: else: logger.warning("No report files uploaded — continuing with debug artifacts") prefix = args.report_prefix or _build_artifact_prefix( - args.submission_name, args.pipeline_run_id, + args.submission_name, + args.pipeline_run_id, ) if args.results_dir: if args.eval_engine == "mcpchecker": @@ -674,12 +683,7 @@ def main() -> int: logger.warning("MinIO not configured — skipping report upload") # --- 2. Quay.io promotion (if enabled) --- - if ( - args.uplift_threshold > -1.0 - and args.recommendation == "pass" - and args.treatment_image_ref - and args.quay_repo - ): + if args.uplift_threshold > -1.0 and args.recommendation == "pass" and args.treatment_image_ref and args.quay_repo: report_path = args.report_dir / "report.json" uplift = 0.0 if report_path.exists(): @@ -702,7 +706,8 @@ def main() -> int: else: logger.info( "Uplift %.4f below threshold %.4f — skipping Quay promotion", - uplift, args.uplift_threshold, + uplift, + args.uplift_threshold, ) else: logger.info("Quay promotion disabled or not applicable") diff --git a/scripts/query_results.py b/scripts/query_results.py index b45b43c..13c6e08 100644 --- a/scripts/query_results.py +++ b/scripts/query_results.py @@ -80,8 +80,7 @@ def cmd_list(session_factory) -> None: select(EvaluationRun) .join( subq, - (EvaluationRun.submission_name == subq.c.submission_name) - & (EvaluationRun.created_at == subq.c.latest), + (EvaluationRun.submission_name == subq.c.submission_name) & (EvaluationRun.created_at == subq.c.latest), ) .order_by(desc(EvaluationRun.created_at)) ) @@ -119,22 +118,30 @@ def cmd_latest(session_factory, name: str) -> None: print(f"Uplift: {run.uplift:.4f}") print() print("Treatment:") - print(f" Trials: {run.treatment_n_trials} " - f"Pass: {run.treatment_n_passed} " - f"Fail: {run.treatment_n_failed} " - f"Errors: {run.treatment_n_errors}") - print(f" Pass rate: {run.treatment_pass_rate:.4f} " - f"Mean reward: {run.treatment_mean_reward or '—'} " - f"Std: {run.treatment_std_reward or '—'}") + print( + f" Trials: {run.treatment_n_trials} " + f"Pass: {run.treatment_n_passed} " + f"Fail: {run.treatment_n_failed} " + f"Errors: {run.treatment_n_errors}" + ) + print( + f" Pass rate: {run.treatment_pass_rate:.4f} " + f"Mean reward: {run.treatment_mean_reward or '—'} " + f"Std: {run.treatment_std_reward or '—'}" + ) print() print("Control:") - print(f" Trials: {run.control_n_trials} " - f"Pass: {run.control_n_passed} " - f"Fail: {run.control_n_failed} " - f"Errors: {run.control_n_errors}") - print(f" Pass rate: {run.control_pass_rate:.4f} " - f"Mean reward: {run.control_mean_reward or '—'} " - f"Std: {run.control_std_reward or '—'}") + print( + f" Trials: {run.control_n_trials} " + f"Pass: {run.control_n_passed} " + f"Fail: {run.control_n_failed} " + f"Errors: {run.control_n_errors}" + ) + print( + f" Pass rate: {run.control_pass_rate:.4f} " + f"Mean reward: {run.control_mean_reward or '—'} " + f"Std: {run.control_std_reward or '—'}" + ) print() print(f"t-test p-value: {run.ttest_p_value or '—'}") print(f"Fisher p-value: {run.fisher_p_value or '—'}") @@ -144,11 +151,7 @@ def cmd_latest(session_factory, name: str) -> None: def cmd_history(session_factory, name: str) -> None: """Show all runs for a submission.""" - stmt = ( - select(EvaluationRun) - .where(EvaluationRun.submission_name == name) - .order_by(desc(EvaluationRun.created_at)) - ) + stmt = select(EvaluationRun).where(EvaluationRun.submission_name == name).order_by(desc(EvaluationRun.created_at)) with session_factory() as session: runs = session.execute(stmt).scalars().all() @@ -164,11 +167,7 @@ def cmd_history(session_factory, name: str) -> None: def cmd_compare(session_factory, name: str) -> None: """Show pass_rate and uplift trend over time.""" - stmt = ( - select(EvaluationRun) - .where(EvaluationRun.submission_name == name) - .order_by(EvaluationRun.created_at) - ) + stmt = select(EvaluationRun).where(EvaluationRun.submission_name == name).order_by(EvaluationRun.created_at) with session_factory() as session: runs = session.execute(stmt).scalars().all() @@ -177,10 +176,7 @@ def cmd_compare(session_factory, name: str) -> None: return print(f"Trend for '{name}' ({len(runs)} runs, oldest first):\n") - print( - f"{'Date':<20} {'Rec':<6} {'Uplift':>8} " - f"{'T.Rate':>7} {'C.Rate':>7} {'T.N':>4} {'C.N':>4}" - ) + print(f"{'Date':<20} {'Rec':<6} {'Uplift':>8} {'T.Rate':>7} {'C.Rate':>7} {'T.N':>4} {'C.N':>4}") print("-" * 75) for r in runs: ts = r.created_at.strftime("%Y-%m-%d %H:%M") if r.created_at else "—" diff --git a/scripts/scaffold.py b/scripts/scaffold.py index 538c005..f94c9ce 100644 --- a/scripts/scaffold.py +++ b/scripts/scaffold.py @@ -145,7 +145,9 @@ def _copy_submission_files( scripts_src = submission_dir / "scripts" if scripts_src.is_dir(): shutil.copytree( - scripts_src, build_context_dir / "scripts", dirs_exist_ok=True, + scripts_src, + build_context_dir / "scripts", + dirs_exist_ok=True, ) @@ -202,7 +204,10 @@ def scaffold_submission( ("control", control_dir), ): context = _build_template_context( - metadata, submission_dir, variant, strategy, + metadata, + submission_dir, + variant, + strategy, ) # Control is a vanilla agent: no supportive files, no scripts, # no CLAUDE.md. Only instruction.md and tests are shared. @@ -273,9 +278,7 @@ def main() -> int: logger.error("Submission directory does not exist: %s", args.submission_dir) return 1 - treatment_dir, control_dir = scaffold_submission( - args.submission_dir, args.output_dir, args.templates_dir - ) + treatment_dir, control_dir = scaffold_submission(args.submission_dir, args.output_dir, args.templates_dir) logger.info("Treatment: %s", treatment_dir) logger.info("Control: %s", control_dir) return 0 diff --git a/scripts/store_results.py b/scripts/store_results.py index 1834338..ee9af3a 100644 --- a/scripts/store_results.py +++ b/scripts/store_results.py @@ -189,9 +189,7 @@ def store_mcpchecker( with session_factory() as session: existing = session.execute( - select(MCPCheckerRun).where( - MCPCheckerRun.pipeline_run_id == effective_run_id - ) + select(MCPCheckerRun).where(MCPCheckerRun.pipeline_run_id == effective_run_id) ).scalar_one_or_none() if existing is not None: @@ -267,9 +265,7 @@ def store( with session_factory() as session: existing = session.execute( - select(EvaluationRun).where( - EvaluationRun.pipeline_run_id == effective_run_id - ) + select(EvaluationRun).where(EvaluationRun.pipeline_run_id == effective_run_id) ).scalar_one_or_none() if existing is not None: @@ -287,9 +283,7 @@ def store( # Check if security scans already exist (may have been inserted by # security-scan task's immediate persistence step) existing_scan = session.execute( - select(SecurityScan).where( - SecurityScan.pipeline_run_id == effective_run_id - ) + select(SecurityScan).where(SecurityScan.pipeline_run_id == effective_run_id) ).scalar_one_or_none() session.add(ev_run) diff --git a/scripts/test_quality_review.py b/scripts/test_quality_review.py index 9f696f1..8d290e1 100644 --- a/scripts/test_quality_review.py +++ b/scripts/test_quality_review.py @@ -13,7 +13,6 @@ import argparse import json import logging -import os import sys from pathlib import Path @@ -176,11 +175,11 @@ def review_submission(submission_dir: Path) -> dict: skill_md_path = _find_skill_md(submission_dir) skill_content = _read_file_safe(skill_md_path) if skill_md_path else "(file not present)" - + # Detect ASE vs Harbor format evals_path = submission_dir / "evals" / "evals.json" is_ase = evals_path.is_file() - + if is_ase: # ASE format: uses evals/evals.json evals_content = _read_file_safe(evals_path) @@ -198,9 +197,7 @@ def review_submission(submission_dir: Path) -> dict: llm_judge_path = submission_dir / "tests" / "llm_judge.py" llm_judge_section = "" if llm_judge_path.is_file(): - llm_judge_section = ( - f"## tests/llm_judge.py\n```python\n{llm_judge_path.read_text()}\n```" - ) + llm_judge_section = f"## tests/llm_judge.py\n```python\n{llm_judge_path.read_text()}\n```" system_prompt = REVIEW_SYSTEM_PROMPT_HARBOR user_prompt = REVIEW_USER_TEMPLATE_HARBOR.format( @@ -224,6 +221,7 @@ def review_submission(submission_dir: Path) -> dict: assessment = json.loads(response_text) except json.JSONDecodeError: import re + json_match = re.search(r"\{.*\}", response_text, re.DOTALL) if json_match: assessment = json.loads(json_match.group()) @@ -273,8 +271,9 @@ def main(argv: list[str] | None = None) -> int: # Always return 0 - this review is advisory (non-blocking) # The assessment result is logged but doesn't fail the pipeline if not assessment.get("passed", False): - logger.warning("Quality review recommendation: %s (advisory, non-blocking)", - assessment.get("recommendation", "unknown")) + logger.warning( + "Quality review recommendation: %s (advisory, non-blocking)", assessment.get("recommendation", "unknown") + ) return 0 diff --git a/scripts/validate.py b/scripts/validate.py index 7865706..843a300 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -112,8 +112,7 @@ def _check_metadata_yaml(submission_dir: Path) -> tuple[list[str], SubmissionMet model = SubmissionMetadata(**raw) except ValidationError as exc: errors = [ - f"metadata.yaml validation: {e['msg']} ({'.'.join(str(loc) for loc in e['loc'])})" - for e in exc.errors() + f"metadata.yaml validation: {e['msg']} ({'.'.join(str(loc) for loc in e['loc'])})" for e in exc.errors() ] return errors, None return [], model @@ -134,6 +133,7 @@ def _check_supportive_size(submission_dir: Path) -> list[str]: # ASE-specific checks # --------------------------------------------------------------------------- + def _check_skill_md_frontmatter(submission_dir: Path) -> list[str]: """Validate that at least one SKILL.md has YAML frontmatter with 'name'.""" skill_files: list[Path] = [] @@ -176,7 +176,7 @@ def _check_skill_md_frontmatter(submission_dir: Path) -> list[str]: def _check_evals_json(submission_dir: Path) -> list[str]: """Validate evals/evals.json for agent-skills-eval format. - + Note: Missing evals.json is NOT an error - the pipeline will generate it from SKILL.md. This function only validates format when the file exists. """ @@ -215,6 +215,7 @@ def _check_evals_json(submission_dir: Path) -> list[str]: # MCPChecker-specific checks # --------------------------------------------------------------------------- + def _check_mcpchecker_eval_yaml(submission_dir: Path) -> list[str]: """Validate eval.yaml exists and has valid MCPChecker structure.""" eval_path = submission_dir / "eval.yaml" @@ -303,6 +304,7 @@ def _check_mcpchecker_tasks(submission_dir: Path) -> list[str]: # Main validation # --------------------------------------------------------------------------- + def validate_submission( submission_dir: Path, eval_engine: EvalEngine = EvalEngine.HARBOR, @@ -351,9 +353,7 @@ def validate_submission( has_task_toml = (submission_dir / "task.toml").is_file() has_tasks_dir = (submission_dir / "tasks").is_dir() if not has_instruction and not has_task_toml and not has_tasks_dir: - errors.append( - "A2A evaluation requires instruction.md, task.toml, or tasks/ directory" - ) + errors.append("A2A evaluation requires instruction.md, task.toml, or tasks/ directory") # Common: supportive/ size check (skip for mcpchecker and a2a) if not run_mcpchecker and not run_a2a: diff --git a/tests/test_ai_review.py b/tests/test_ai_review.py deleted file mode 100644 index e3b28b8..0000000 --- a/tests/test_ai_review.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Tests for scripts/ai_review.py.""" - -from __future__ import annotations - -import json -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest -import yaml - -from scripts.ai_review import main, review_submission - -VALID_METADATA = { - "name": "review-skill", - "description": "A skill to review", - "generation_mode": "manual", -} - -MOCK_PASS_RESPONSE = json.dumps({ - "dimensions": { - "coherence": {"score": 0.9, "finding": "Instruction matches skill well."}, - "coverage": {"score": 0.8, "finding": "Tests cover main requirements."}, - "clarity": {"score": 0.85, "finding": "Clear and unambiguous."}, - "feasibility": {"score": 0.9, "finding": "Achievable in one session."}, - "robustness": {"score": 0.7, "finding": "Some edge cases missing."}, - }, - "overall_score": 0.83, - "recommendation": "pass", - "summary": "Good quality submission ready for evaluation.", -}) - -MOCK_FAIL_RESPONSE = json.dumps({ - "dimensions": { - "coherence": {"score": 0.3, "finding": "Instruction does not match skill."}, - "coverage": {"score": 0.2, "finding": "Tests are trivial."}, - "clarity": {"score": 0.5, "finding": "Some ambiguity."}, - "feasibility": {"score": 0.4, "finding": "May be too complex."}, - "robustness": {"score": 0.2, "finding": "No edge cases."}, - }, - "overall_score": 0.32, - "recommendation": "fail", - "summary": "Significant quality issues prevent evaluation.", -}) - - -@pytest.fixture() -def review_submission_dir(tmp_path: Path) -> Path: - sub = tmp_path / "review-skill" - sub.mkdir() - (sub / "metadata.yaml").write_text(yaml.dump(VALID_METADATA)) - (sub / "skills").mkdir() - (sub / "skills" / "SKILL.md").write_text("# Skill\nDo code review.\n") - (sub / "instruction.md").write_text("# Task\nReview the code.\n") - (sub / "tests").mkdir() - (sub / "tests" / "test_outputs.py").write_text("def test_review(): assert True\n") - return sub - - -class TestReviewSubmission: - @patch("scripts.ai_review.llm_client.chat_completion") - def test_passing_review( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - ) -> None: - mock_chat.return_value = MOCK_PASS_RESPONSE - - result = review_submission(review_submission_dir) - - assert result["passed"] is True - assert result["recommendation"] == "pass" - assert result["overall_score"] > 0.6 - assert "dimensions" in result - assert "coherence" in result["dimensions"] - - @patch("scripts.ai_review.llm_client.chat_completion") - def test_failing_review( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - ) -> None: - mock_chat.return_value = MOCK_FAIL_RESPONSE - - result = review_submission(review_submission_dir) - - assert result["passed"] is False - assert result["recommendation"] == "fail" - assert result["overall_score"] < 0.4 - - @patch("scripts.ai_review.llm_client.chat_completion") - def test_missing_overall_score_computed( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - ) -> None: - response_no_score = { - "dimensions": { - "coherence": {"score": 0.8, "finding": "Good."}, - "coverage": {"score": 0.7, "finding": "Ok."}, - "clarity": {"score": 0.9, "finding": "Clear."}, - "feasibility": {"score": 0.8, "finding": "Doable."}, - "robustness": {"score": 0.6, "finding": "Basic."}, - }, - } - mock_chat.return_value = json.dumps(response_no_score) - - result = review_submission(review_submission_dir) - - assert "overall_score" in result - assert 0.7 < result["overall_score"] < 0.9 - assert result["recommendation"] == "pass" - assert result["passed"] is True - - @patch("scripts.ai_review.llm_client.chat_completion") - def test_includes_llm_judge_when_present( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - ) -> None: - (review_submission_dir / "tests" / "llm_judge.py").write_text("score = 0.9\n") - mock_chat.return_value = MOCK_PASS_RESPONSE - - review_submission(review_submission_dir) - - call_args = mock_chat.call_args - messages = call_args.args[0] if call_args.args else call_args.kwargs["messages"] - user_msg = messages[1]["content"] - assert "llm_judge.py" in user_msg - - @patch("scripts.ai_review.llm_client.chat_completion") - def test_handles_missing_files( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - ) -> None: - (review_submission_dir / "instruction.md").unlink() - mock_chat.return_value = MOCK_PASS_RESPONSE - - result = review_submission(review_submission_dir) - assert result["passed"] is True - - @patch("scripts.ai_review.llm_client.chat_completion") - def test_invalid_json_response_raises( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - ) -> None: - mock_chat.return_value = "Not JSON at all, no braces here" - - with pytest.raises(ValueError, match="not valid JSON"): - review_submission(review_submission_dir) - - -class TestMain: - @patch("scripts.ai_review.llm_client.chat_completion") - def test_pass_returns_zero( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - mock_chat.return_value = MOCK_PASS_RESPONSE - rc = main([str(review_submission_dir)]) - assert rc == 0 - output = json.loads(capsys.readouterr().out) - assert output["passed"] is True - - @patch("scripts.ai_review.llm_client.chat_completion") - def test_fail_returns_one( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - mock_chat.return_value = MOCK_FAIL_RESPONSE - rc = main([str(review_submission_dir)]) - assert rc == 1 - output = json.loads(capsys.readouterr().out) - assert output["passed"] is False - - def test_nonexistent_dir_returns_one( - self, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - rc = main([str(tmp_path / "nope")]) - assert rc == 1 - output = json.loads(capsys.readouterr().out) - assert output["passed"] is False - - @patch("scripts.ai_review.llm_client.chat_completion") - def test_api_error_returns_one( - self, - mock_chat: MagicMock, - review_submission_dir: Path, - capsys: pytest.CaptureFixture[str], - ) -> None: - mock_chat.side_effect = RuntimeError("API down") - rc = main([str(review_submission_dir)]) - assert rc == 1 - output = json.loads(capsys.readouterr().out) - assert "error" in output diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 07ff905..58514c9 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -29,11 +29,11 @@ render_markdown, ) - # --------------------------------------------------------------------------- # Fixtures — fake result directories # --------------------------------------------------------------------------- + def _write_result(trial_dir: Path, reward: float | None, nested: bool = True) -> None: """Write a result.json file to a trial directory.""" trial_dir.mkdir(parents=True, exist_ok=True) @@ -87,6 +87,7 @@ def empty_results_dir(tmp_path: Path) -> Path: # TestExtractReward # --------------------------------------------------------------------------- + class TestExtractReward: def test_nested_format(self): data = {"verifier_result": {"rewards": {"reward": 0.75}}} @@ -124,6 +125,7 @@ def test_integer_reward_cast(self): # TestParseVariantTrials # --------------------------------------------------------------------------- + class TestParseVariantTrials: def test_parses_all_trials(self, results_dir: Path): trials = parse_variant_trials(results_dir / "treatment") @@ -177,9 +179,7 @@ def test_job_level_result_filtered(self, tmp_path: Path): (job_dir / "result.json").write_text(json.dumps({"job_name": "sub-treatment"})) trial_dir = job_dir / "task__001" trial_dir.mkdir() - (trial_dir / "result.json").write_text( - json.dumps({"verifier_result": {"rewards": {"reward": 1.0}}}) - ) + (trial_dir / "result.json").write_text(json.dumps({"verifier_result": {"rewards": {"reward": 1.0}}})) trials = parse_variant_trials(variant) assert len(trials) == 1 assert trials[0].reward == 1.0 @@ -189,6 +189,7 @@ def test_job_level_result_filtered(self, tmp_path: Path): # TestComputeVariantSummary # --------------------------------------------------------------------------- + class TestComputeVariantSummary: def test_basic_stats(self, results_dir: Path): trials = parse_variant_trials(results_dir / "treatment") @@ -237,6 +238,7 @@ def test_all_zero_rewards(self): # TestStatisticalTests # --------------------------------------------------------------------------- + class TestStatisticalTests: def test_ttest_significant_difference(self, results_dir: Path): t_trials = parse_variant_trials(results_dir / "treatment") @@ -301,6 +303,7 @@ def test_fisher_excludes_error_trials(self): # TestBuildAnalysis # --------------------------------------------------------------------------- + class TestBuildAnalysis: def test_full_analysis(self, results_dir: Path): result = build_analysis(results_dir, "my-submission") @@ -333,9 +336,13 @@ def test_all_zero_passes_is_fail(self, tmp_path: Path): job = base / variant / f"sub-{variant}" trial = job / "trial-0__abc" trial.mkdir(parents=True) - (trial / "result.json").write_text(json.dumps({ - "verifier_result": {"rewards": {"reward": 0.0}}, - })) + (trial / "result.json").write_text( + json.dumps( + { + "verifier_result": {"rewards": {"reward": 0.0}}, + } + ) + ) result = build_analysis(base, "all-fail", threshold=0.0) assert result.summary.treatment.n_passed == 0 assert result.summary.control.n_passed == 0 @@ -365,6 +372,7 @@ def test_mean_reward_gap(self, results_dir: Path): def test_small_sample_warning(self, results_dir: Path, caplog): import logging + with caplog.at_level(logging.WARNING): build_analysis(results_dir, "my-submission") assert any("only 5 trials" in msg for msg in caplog.messages) @@ -374,6 +382,7 @@ def test_small_sample_warning(self, results_dir: Path, caplog): # TestRenderMarkdown # --------------------------------------------------------------------------- + class TestRenderMarkdown: def test_contains_title(self, results_dir: Path): result = build_analysis(results_dir, "my-submission") @@ -421,6 +430,7 @@ def test_significance_markers(self, results_dir: Path): # TestJsonRoundtrip # --------------------------------------------------------------------------- + class TestJsonRoundtrip: def test_json_serialization(self, results_dir: Path): result = build_analysis(results_dir, "my-submission") @@ -435,14 +445,20 @@ def test_json_serialization(self, results_dir: Path): # TestMainCLI # --------------------------------------------------------------------------- + class TestMainCLI: def test_success(self, results_dir: Path, tmp_path: Path): out_dir = tmp_path / "reports" - rc = main([ - "--results-dir", str(results_dir), - "--output-dir", str(out_dir), - "--submission-name", "my-submission", - ]) + rc = main( + [ + "--results-dir", + str(results_dir), + "--output-dir", + str(out_dir), + "--submission-name", + "my-submission", + ] + ) assert rc == 0 assert (out_dir / "report.json").is_file() assert (out_dir / "report.md").is_file() @@ -452,57 +468,88 @@ def test_success(self, results_dir: Path, tmp_path: Path): def test_with_provenance_flags(self, results_dir: Path, tmp_path: Path): out_dir = tmp_path / "reports" - rc = main([ - "--results-dir", str(results_dir), - "--output-dir", str(out_dir), - "--submission-name", "my-submission", - "--commit-sha", "abc123", - "--pipeline-run-id", "run-42", - "--treatment-image-ref", "img@sha256:aaa", - "--control-image-ref", "img@sha256:bbb", - "--harbor-fork-revision", "main", - ]) + rc = main( + [ + "--results-dir", + str(results_dir), + "--output-dir", + str(out_dir), + "--submission-name", + "my-submission", + "--commit-sha", + "abc123", + "--pipeline-run-id", + "run-42", + "--treatment-image-ref", + "img@sha256:aaa", + "--control-image-ref", + "img@sha256:bbb", + "--harbor-fork-revision", + "main", + ] + ) assert rc == 0 report = json.loads((out_dir / "report.json").read_text()) assert report["provenance"]["commit_sha"] == "abc123" def test_with_threshold(self, results_dir: Path, tmp_path: Path): out_dir = tmp_path / "reports" - rc = main([ - "--results-dir", str(results_dir), - "--output-dir", str(out_dir), - "--submission-name", "my-submission", - "--threshold", "0.9", - ]) + rc = main( + [ + "--results-dir", + str(results_dir), + "--output-dir", + str(out_dir), + "--submission-name", + "my-submission", + "--threshold", + "0.9", + ] + ) assert rc == 0 report = json.loads((out_dir / "report.json").read_text()) assert report["summary"]["recommendation"] == "fail" def test_nonexistent_results_dir(self, tmp_path: Path): - rc = main([ - "--results-dir", str(tmp_path / "nope"), - "--output-dir", str(tmp_path / "out"), - "--submission-name", "x", - ]) + rc = main( + [ + "--results-dir", + str(tmp_path / "nope"), + "--output-dir", + str(tmp_path / "out"), + "--submission-name", + "x", + ] + ) assert rc == 1 def test_creates_output_dir(self, results_dir: Path, tmp_path: Path): out_dir = tmp_path / "nested" / "dir" / "reports" - rc = main([ - "--results-dir", str(results_dir), - "--output-dir", str(out_dir), - "--submission-name", "my-submission", - ]) + rc = main( + [ + "--results-dir", + str(results_dir), + "--output-dir", + str(out_dir), + "--submission-name", + "my-submission", + ] + ) assert rc == 0 assert (out_dir / "report.json").is_file() def test_markdown_written(self, results_dir: Path, tmp_path: Path): out_dir = tmp_path / "reports" - main([ - "--results-dir", str(results_dir), - "--output-dir", str(out_dir), - "--submission-name", "my-submission", - ]) + main( + [ + "--results-dir", + str(results_dir), + "--output-dir", + str(out_dir), + "--submission-name", + "my-submission", + ] + ) md = (out_dir / "report.md").read_text() assert "# A/B Evaluation Report" in md @@ -511,6 +558,7 @@ def test_markdown_written(self, results_dir: Path, tmp_path: Path): # TestSecurityModels # --------------------------------------------------------------------------- + class TestSecurityModels: """Tests for SecurityFinding and SecurityScanResult Pydantic models.""" diff --git a/tests/test_ase_integration.py b/tests/test_ase_integration.py index cc7c2a8..4f1ccba 100644 --- a/tests/test_ase_integration.py +++ b/tests/test_ase_integration.py @@ -12,11 +12,11 @@ import pytest import yaml -from abevalflow.schemas import EvalEngine, SubmissionMetadata from abevalflow.report import AnalysisResult, Provenance -from scripts.validate import validate_submission, main as validate_main +from abevalflow.schemas import EvalEngine, SubmissionMetadata from scripts.aggregate_ase import build_ase_analysis, render_markdown - +from scripts.validate import main as validate_main +from scripts.validate import validate_submission # --------------------------------------------------------------------------- # Fixtures @@ -113,47 +113,59 @@ def ase_results_dir(tmp_path: Path) -> Path: ws = skill_dir / "with_skill" ws.mkdir(parents=True) pass_rate = 0.8 + (i * 0.04) - (ws / "grading.json").write_text(json.dumps({ - "assertion_results": [ - {"text": "A1", "passed": True, "evidence": "ok"}, - {"text": "A2", "passed": True, "evidence": "ok"}, - {"text": "A3", "passed": True, "evidence": "ok"}, - {"text": "A4", "passed": i != 2, "evidence": "ok"}, - {"text": "A5", "passed": i == 3, "evidence": "ok"}, - ], - "summary": { - "passed": 3 + (1 if i != 2 else 0) + (1 if i == 3 else 0), - "failed": 5 - (3 + (1 if i != 2 else 0) + (1 if i == 3 else 0)), - "total": 5, - "pass_rate": pass_rate, - }, - })) + (ws / "grading.json").write_text( + json.dumps( + { + "assertion_results": [ + {"text": "A1", "passed": True, "evidence": "ok"}, + {"text": "A2", "passed": True, "evidence": "ok"}, + {"text": "A3", "passed": True, "evidence": "ok"}, + {"text": "A4", "passed": i != 2, "evidence": "ok"}, + {"text": "A5", "passed": i == 3, "evidence": "ok"}, + ], + "summary": { + "passed": 3 + (1 if i != 2 else 0) + (1 if i == 3 else 0), + "failed": 5 - (3 + (1 if i != 2 else 0) + (1 if i == 3 else 0)), + "total": 5, + "pass_rate": pass_rate, + }, + } + ) + ) wos = skill_dir / "without_skill" wos.mkdir(parents=True) - (wos / "grading.json").write_text(json.dumps({ - "assertion_results": [ - {"text": "A1", "passed": True, "evidence": "ok"}, - {"text": "A2", "passed": False, "evidence": "no"}, - {"text": "A3", "passed": False, "evidence": "no"}, - {"text": "A4", "passed": False, "evidence": "no"}, - {"text": "A5", "passed": False, "evidence": "no"}, - ], - "summary": { - "passed": 1, - "failed": 4, - "total": 5, - "pass_rate": 0.2, - }, - })) - - (iter_dir / "benchmark.json").write_text(json.dumps({ - "run_summary": { - "with_skill": {"pass_rate": {"mean": pass_rate, "stddev": 0}}, - "without_skill": {"pass_rate": {"mean": 0.2, "stddev": 0}}, - "delta": {"pass_rate": pass_rate - 0.2}, - }, - })) + (wos / "grading.json").write_text( + json.dumps( + { + "assertion_results": [ + {"text": "A1", "passed": True, "evidence": "ok"}, + {"text": "A2", "passed": False, "evidence": "no"}, + {"text": "A3", "passed": False, "evidence": "no"}, + {"text": "A4", "passed": False, "evidence": "no"}, + {"text": "A5", "passed": False, "evidence": "no"}, + ], + "summary": { + "passed": 1, + "failed": 4, + "total": 5, + "pass_rate": 0.2, + }, + } + ) + ) + + (iter_dir / "benchmark.json").write_text( + json.dumps( + { + "run_summary": { + "with_skill": {"pass_rate": {"mean": pass_rate, "stddev": 0}}, + "without_skill": {"pass_rate": {"mean": 0.2, "stddev": 0}}, + "delta": {"pass_rate": pass_rate - 0.2}, + }, + } + ) + ) return results diff --git a/tests/test_certification.py b/tests/test_certification.py index ca5bc10..b0461af 100644 --- a/tests/test_certification.py +++ b/tests/test_certification.py @@ -240,8 +240,7 @@ def test_validation_failure_affects_certification(self) -> None: has_eval_assets=True, ) valid_structure_check = next( - c for c in result.foundational.checks - if c.check_id == CheckId.VALID_SKILL_STRUCTURE + c for c in result.foundational.checks if c.check_id == CheckId.VALID_SKILL_STRUCTURE ) assert valid_structure_check.passed is False @@ -252,10 +251,7 @@ def test_missing_metadata_affects_certification(self) -> None: metadata_valid=False, has_eval_assets=True, ) - metadata_check = next( - c for c in result.foundational.checks - if c.check_id == CheckId.METADATA_COMPLIANCE - ) + metadata_check = next(c for c in result.foundational.checks if c.check_id == CheckId.METADATA_COMPLIANCE) assert metadata_check.passed is False def test_missing_eval_assets_affects_trusted(self) -> None: @@ -265,10 +261,7 @@ def test_missing_eval_assets_affects_trusted(self) -> None: metadata_valid=True, has_eval_assets=False, ) - eval_assets_check = next( - c for c in result.trusted.checks - if c.check_id == CheckId.EVALUATION_ASSETS - ) + eval_assets_check = next(c for c in result.trusted.checks if c.check_id == CheckId.EVALUATION_ASSETS) assert eval_assets_check.passed is False def test_high_score_engine_provides_advanced_validation(self) -> None: @@ -285,10 +278,7 @@ def test_high_score_engine_provides_advanced_validation(self) -> None: metadata_valid=True, has_eval_assets=True, ) - advanced_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION - ) + advanced_check = next(c for c in result.certified.checks if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION) assert advanced_check.passed is True def test_low_score_engine_fails_advanced_validation(self) -> None: @@ -305,10 +295,7 @@ def test_low_score_engine_fails_advanced_validation(self) -> None: metadata_valid=True, has_eval_assets=True, ) - advanced_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION - ) + advanced_check = next(c for c in result.certified.checks if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION) assert advanced_check.passed is False def test_security_findings_affect_enterprise_review(self) -> None: @@ -333,10 +320,7 @@ def test_security_findings_affect_enterprise_review(self) -> None: metadata_valid=True, has_eval_assets=True, ) - enterprise_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW - ) + enterprise_check = next(c for c in result.certified.checks if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW) assert enterprise_check.passed is False def test_default_checks_are_valid_check_ids(self) -> None: @@ -429,15 +413,11 @@ def test_custom_checks_for_foundational(self) -> None: def test_custom_checks_for_certified(self) -> None: """Custom check list for certified level respects hierarchy. - + Even if certified's own checks pass, certified.passed is False if foundational or trusted fail (hierarchy enforcement). """ - policy = CertificationPolicy( - certified=CertificationLevelPolicy( - checks=["advanced_agent_validation"] - ) - ) + policy = CertificationPolicy(certified=CertificationLevelPolicy(checks=["advanced_agent_validation"])) engine_gate = GateResult( gate_name="evaluation", gate_type=GateType.ENGINE, @@ -462,11 +442,7 @@ def test_custom_checks_for_certified(self) -> None: def test_threshold_override_for_advanced_agent_validation(self) -> None: """Override threshold for advanced_agent_validation.""" - policy = CertificationPolicy( - certified=CertificationLevelPolicy( - thresholds={"advanced_agent_validation": 0.5} - ) - ) + policy = CertificationPolicy(certified=CertificationLevelPolicy(thresholds={"advanced_agent_validation": 0.5})) engine_gate = GateResult( gate_name="evaluation", gate_type=GateType.ENGINE, @@ -481,19 +457,12 @@ def test_threshold_override_for_advanced_agent_validation(self) -> None: has_eval_assets=True, policy=policy, ) - advanced_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION - ) + advanced_check = next(c for c in result.certified.checks if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION) assert advanced_check.passed is True def test_threshold_override_fails_when_below(self) -> None: """Score below overridden threshold should fail.""" - policy = CertificationPolicy( - certified=CertificationLevelPolicy( - thresholds={"advanced_agent_validation": 0.9} - ) - ) + policy = CertificationPolicy(certified=CertificationLevelPolicy(thresholds={"advanced_agent_validation": 0.9})) engine_gate = GateResult( gate_name="evaluation", gate_type=GateType.ENGINE, @@ -508,10 +477,7 @@ def test_threshold_override_fails_when_below(self) -> None: has_eval_assets=True, policy=policy, ) - advanced_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION - ) + advanced_check = next(c for c in result.certified.checks if c.check_id == CheckId.ADVANCED_AGENT_VALIDATION) assert advanced_check.passed is False assert "0.90" in advanced_check.message @@ -545,11 +511,7 @@ def test_invalid_check_id_raises_error(self) -> None: def test_threshold_override_for_instruction_quality(self) -> None: """Override threshold for instruction_quality check.""" - policy = CertificationPolicy( - trusted=CertificationLevelPolicy( - thresholds={"instruction_quality": 0.5} - ) - ) + policy = CertificationPolicy(trusted=CertificationLevelPolicy(thresholds={"instruction_quality": 0.5})) quality_gate = GateResult( gate_name="quality", gate_type=GateType.QUALITY, @@ -564,19 +526,12 @@ def test_threshold_override_for_instruction_quality(self) -> None: has_eval_assets=True, policy=policy, ) - instruction_check = next( - c for c in result.trusted.checks - if c.check_id == CheckId.INSTRUCTION_QUALITY - ) + instruction_check = next(c for c in result.trusted.checks if c.check_id == CheckId.INSTRUCTION_QUALITY) assert instruction_check.passed is True def test_threshold_override_for_advanced_security(self) -> None: """Override threshold for advanced_security_validation check.""" - policy = CertificationPolicy( - trusted=CertificationLevelPolicy( - thresholds={"advanced_security_validation": 0.7} - ) - ) + policy = CertificationPolicy(trusted=CertificationLevelPolicy(thresholds={"advanced_security_validation": 0.7})) security_gate = GateResult( gate_name="security", gate_type=GateType.SECURITY, @@ -593,16 +548,13 @@ def test_threshold_override_for_advanced_security(self) -> None: policy=policy, ) adv_security_check = next( - c for c in result.trusted.checks - if c.check_id == CheckId.ADVANCED_SECURITY_VALIDATION + c for c in result.trusted.checks if c.check_id == CheckId.ADVANCED_SECURITY_VALIDATION ) assert adv_security_check.passed is True def test_custom_checks_empty_list(self) -> None: """Empty check list should fail the level (no checks = cannot pass).""" - policy = CertificationPolicy( - foundational=CertificationLevelPolicy(checks=[]) - ) + policy = CertificationPolicy(foundational=CertificationLevelPolicy(checks=[])) result = compute_certification( gates=[], validation_passed=True, @@ -663,16 +615,12 @@ class TestInvalidThresholdKeyValidation: def test_invalid_threshold_key_raises_error(self) -> None: """Invalid threshold key should raise ValueError during validation.""" with pytest.raises(ValueError) as exc_info: - CertificationLevelPolicy( - thresholds={"invalid_check_name": 0.5} - ) + CertificationLevelPolicy(thresholds={"invalid_check_name": 0.5}) assert "Invalid threshold key 'invalid_check_name'" in str(exc_info.value) def test_valid_threshold_key_passes(self) -> None: """Valid threshold key should pass validation.""" - policy = CertificationLevelPolicy( - thresholds={"advanced_agent_validation": 0.5} - ) + policy = CertificationLevelPolicy(thresholds={"advanced_agent_validation": 0.5}) assert policy.thresholds["advanced_agent_validation"] == 0.5 def test_multiple_valid_threshold_keys(self) -> None: @@ -717,10 +665,7 @@ def test_message_when_score_below_threshold_no_findings(self) -> None: metadata_valid=True, has_eval_assets=True, ) - enterprise_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW - ) + enterprise_check = next(c for c in result.certified.checks if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW) assert enterprise_check.passed is False assert "Score" in enterprise_check.message assert "below threshold" in enterprise_check.message @@ -741,10 +686,7 @@ def test_message_when_score_high_no_findings(self) -> None: metadata_valid=True, has_eval_assets=True, ) - enterprise_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW - ) + enterprise_check = next(c for c in result.certified.checks if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW) assert enterprise_check.passed is True assert enterprise_check.message == "No security findings" @@ -768,10 +710,7 @@ def test_message_when_findings_present(self) -> None: metadata_valid=True, has_eval_assets=True, ) - enterprise_check = next( - c for c in result.certified.checks - if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW - ) + enterprise_check = next(c for c in result.certified.checks if c.check_id == CheckId.ENTERPRISE_SECURITY_REVIEW) assert enterprise_check.passed is False assert "2 findings require review" in enterprise_check.message @@ -782,12 +721,8 @@ class TestCertificationPolicyGetThreshold: def test_get_threshold_last_wins(self) -> None: """Later level threshold should override earlier level.""" policy = CertificationPolicy( - foundational=CertificationLevelPolicy( - thresholds={"advanced_agent_validation": 0.5} - ), - certified=CertificationLevelPolicy( - thresholds={"advanced_agent_validation": 0.9} - ), + foundational=CertificationLevelPolicy(thresholds={"advanced_agent_validation": 0.5}), + certified=CertificationLevelPolicy(thresholds={"advanced_agent_validation": 0.9}), ) assert policy.get_threshold("advanced_agent_validation") == 0.9 @@ -799,9 +734,7 @@ def test_get_threshold_returns_none_when_not_set(self) -> None: def test_get_threshold_from_single_level(self) -> None: """Should return threshold from whichever level defines it.""" policy = CertificationPolicy( - trusted=CertificationLevelPolicy( - thresholds={"instruction_quality": 0.6} - ), + trusted=CertificationLevelPolicy(thresholds={"instruction_quality": 0.6}), ) assert policy.get_threshold("instruction_quality") == 0.6 @@ -835,10 +768,7 @@ def test_both_mode_keeps_failing_check_conservative(self) -> None: metadata_valid=True, has_eval_assets=True, ) - exec_check = next( - c for c in result.foundational.checks - if c.check_id == CheckId.BASIC_EXECUTION_VALIDATION - ) + exec_check = next(c for c in result.foundational.checks if c.check_id == CheckId.BASIC_EXECUTION_VALIDATION) assert exec_check.passed is False assert exec_check.details.get("source_implementation") == "ase" @@ -866,10 +796,7 @@ def test_both_mode_passing_first_failing_second(self) -> None: metadata_valid=True, has_eval_assets=True, ) - func_check = next( - c for c in result.trusted.checks - if c.check_id == CheckId.FUNCTIONAL_VALIDATION - ) + func_check = next(c for c in result.trusted.checks if c.check_id == CheckId.FUNCTIONAL_VALIDATION) assert func_check.passed is False assert func_check.details.get("source_implementation") == "ase" @@ -897,10 +824,7 @@ def test_both_mode_failing_first_passing_second(self) -> None: metadata_valid=True, has_eval_assets=True, ) - exec_check = next( - c for c in result.foundational.checks - if c.check_id == CheckId.BASIC_EXECUTION_VALIDATION - ) + exec_check = next(c for c in result.foundational.checks if c.check_id == CheckId.BASIC_EXECUTION_VALIDATION) assert exec_check.passed is False assert exec_check.details.get("source_implementation") == "harbor" @@ -920,24 +844,21 @@ def test_check_result_includes_source_implementation(self) -> None: metadata_valid=True, has_eval_assets=True, ) - exec_check = next( - c for c in result.foundational.checks - if c.check_id == CheckId.BASIC_EXECUTION_VALIDATION - ) + exec_check = next(c for c in result.foundational.checks if c.check_id == CheckId.BASIC_EXECUTION_VALIDATION) assert "source_implementation" in exec_check.details assert exec_check.details["source_implementation"] == "harbor" class TestHierarchyEnforcement: """Tests for certification hierarchy enforcement in compute_certification. - + The hierarchy is: Foundational < Trusted < Certified If a lower level fails, higher levels must also fail even if their own checks pass. """ def test_trusted_fails_certified_cascades(self) -> None: """If trusted's own checks fail, certified should also fail. - + Scenario: foundational passes, trusted fails its own checks Expected: certified.passed should be False """ @@ -952,17 +873,11 @@ def test_trusted_fails_certified_cascades(self) -> None: # trusted needs functional_validation which will pass from engine gate, # BUT also needs advanced_security_validation which won't be satisfied policy = CertificationPolicy( - foundational=CertificationLevelPolicy( - checks=["valid_skill_structure", "metadata_compliance"] - ), - trusted=CertificationLevelPolicy( - checks=["functional_validation", "advanced_security_validation"] - ), - certified=CertificationLevelPolicy( - checks=["advanced_agent_validation"] - ), + foundational=CertificationLevelPolicy(checks=["valid_skill_structure", "metadata_compliance"]), + trusted=CertificationLevelPolicy(checks=["functional_validation", "advanced_security_validation"]), + certified=CertificationLevelPolicy(checks=["advanced_agent_validation"]), ) - + result = compute_certification( gates=[engine_gate], validation_passed=True, @@ -970,7 +885,7 @@ def test_trusted_fails_certified_cascades(self) -> None: has_eval_assets=True, policy=policy, ) - + # Foundational should pass (only needs structure/metadata) assert result.foundational.passed is True # Trusted should fail (advanced_security_validation not satisfied) @@ -981,7 +896,7 @@ def test_trusted_fails_certified_cascades(self) -> None: def test_foundational_fails_trusted_cascades(self) -> None: """If foundational fails, trusted should also fail even if its own checks pass. - + Scenario: foundational fails (validation_passed=False), provide gates for trusted Expected: trusted.passed should be False despite its checks being satisfied """ @@ -1000,14 +915,14 @@ def test_foundational_fails_trusted_cascades(self) -> None: score=0.85, mode=GateMode.BLOCK, ) - + result = compute_certification( gates=[security_gate, engine_gate], validation_passed=False, # This causes foundational to fail metadata_valid=True, has_eval_assets=True, ) - + # Foundational fails because validation_passed=False assert result.foundational.passed is False # Trusted should fail due to hierarchy even if its checks might pass @@ -1018,7 +933,7 @@ def test_foundational_fails_trusted_cascades(self) -> None: def test_foundational_fails_trusted_checks_still_preserved(self) -> None: """When hierarchy forces trusted to fail, individual check results are preserved. - + This allows consumers to see what would have passed if foundational passed. """ # Provide gates that satisfy trusted checks @@ -1036,21 +951,18 @@ def test_foundational_fails_trusted_checks_still_preserved(self) -> None: score=0.85, mode=GateMode.BLOCK, ) - + result = compute_certification( gates=[security_gate, engine_gate], validation_passed=False, # Causes foundational to fail metadata_valid=True, has_eval_assets=True, ) - + # Trusted level fails due to hierarchy assert result.trusted.passed is False # But individual checks that could be evaluated are still present - func_check = next( - (c for c in result.trusted.checks if c.check_id == CheckId.FUNCTIONAL_VALIDATION), - None - ) + func_check = next((c for c in result.trusted.checks if c.check_id == CheckId.FUNCTIONAL_VALIDATION), None) assert func_check is not None # The check itself passed (from the engine gate) assert func_check.passed is True @@ -1080,17 +992,11 @@ def test_all_levels_pass_when_all_requirements_met(self) -> None: ) # Use policy to simplify check requirements policy = CertificationPolicy( - foundational=CertificationLevelPolicy( - checks=["valid_skill_structure", "metadata_compliance"] - ), - trusted=CertificationLevelPolicy( - checks=["functional_validation"] - ), - certified=CertificationLevelPolicy( - checks=["advanced_agent_validation"] - ), + foundational=CertificationLevelPolicy(checks=["valid_skill_structure", "metadata_compliance"]), + trusted=CertificationLevelPolicy(checks=["functional_validation"]), + certified=CertificationLevelPolicy(checks=["advanced_agent_validation"]), ) - + result = compute_certification( gates=[security_gate, quality_gate, engine_gate], validation_passed=True, @@ -1098,7 +1004,7 @@ def test_all_levels_pass_when_all_requirements_met(self) -> None: has_eval_assets=True, policy=policy, ) - + assert result.foundational.passed is True assert result.trusted.passed is True assert result.certified.passed is True @@ -1111,10 +1017,10 @@ class TestProfileLoading: def test_load_profile_skill(self) -> None: """Load the default 'skill' profile.""" from abevalflow.certification import clear_profiles_cache, load_profile - + clear_profiles_cache() policy = load_profile("skill") - + assert policy is not None assert policy.foundational is not None assert "valid_skill_structure" in policy.foundational.checks @@ -1123,10 +1029,10 @@ def test_load_profile_skill(self) -> None: def test_load_profile_agent(self) -> None: """Load the 'agent' profile with agent-specific checks.""" from abevalflow.certification import clear_profiles_cache, load_profile - + clear_profiles_cache() policy = load_profile("agent") - + assert policy is not None assert policy.certified is not None assert "advanced_agent_validation" in policy.certified.checks @@ -1135,10 +1041,10 @@ def test_load_profile_agent(self) -> None: def test_load_profile_mcp_server(self) -> None: """Load the 'mcp_server' profile with API-focused checks.""" from abevalflow.certification import clear_profiles_cache, load_profile - + clear_profiles_cache() policy = load_profile("mcp_server") - + assert policy is not None assert policy.certified is not None assert "enterprise_security_review" in policy.certified.checks @@ -1146,7 +1052,7 @@ def test_load_profile_mcp_server(self) -> None: def test_load_profile_invalid_raises(self) -> None: """Loading a non-existent profile raises ValueError.""" from abevalflow.certification import clear_profiles_cache, load_profile - + clear_profiles_cache() with pytest.raises(ValueError, match="Unknown certification profile"): load_profile("nonexistent_profile") @@ -1154,10 +1060,10 @@ def test_load_profile_invalid_raises(self) -> None: def test_get_available_profiles(self) -> None: """Get list of available profile names.""" from abevalflow.certification import clear_profiles_cache, get_available_profiles - + clear_profiles_cache() profiles = get_available_profiles() - + assert "skill" in profiles assert "agent" in profiles assert "mcp_server" in profiles @@ -1166,19 +1072,19 @@ def test_get_available_profiles(self) -> None: def test_get_default_profile_name(self) -> None: """Get the default profile name from configuration.""" from abevalflow.certification import clear_profiles_cache, get_default_profile_name - + clear_profiles_cache() default = get_default_profile_name() - + assert default == "skill" def test_load_profile_none_uses_default(self) -> None: """Loading with None uses the default profile.""" from abevalflow.certification import clear_profiles_cache, load_profile - + clear_profiles_cache() policy = load_profile(None) - + # Should load 'skill' profile (the default) assert policy is not None assert policy.foundational is not None @@ -1187,10 +1093,10 @@ def test_load_profile_none_uses_default(self) -> None: def test_profile_integrates_with_compute_certification(self) -> None: """Profile-loaded policy works with compute_certification.""" from abevalflow.certification import clear_profiles_cache, load_profile - + clear_profiles_cache() policy = load_profile("skill") - + engine_gate = GateResult( gate_name="evaluation", gate_type=GateType.ENGINE, @@ -1198,7 +1104,7 @@ def test_profile_integrates_with_compute_certification(self) -> None: score=0.9, mode=GateMode.BLOCK, ) - + result = compute_certification( gates=[engine_gate], validation_passed=True, @@ -1206,7 +1112,7 @@ def test_profile_integrates_with_compute_certification(self) -> None: has_eval_assets=True, policy=policy, ) - + # Should use skill profile's check configuration assert result is not None assert result.foundational is not None diff --git a/tests/test_compass_facts.py b/tests/test_compass_facts.py index b90c831..fb24be4 100644 --- a/tests/test_compass_facts.py +++ b/tests/test_compass_facts.py @@ -2,7 +2,7 @@ import json import logging -from datetime import datetime, timezone +from datetime import UTC, datetime from unittest.mock import MagicMock, patch import pytest @@ -15,7 +15,6 @@ LevelResult, ) from abevalflow.compass_facts import ( - CertificationFactPushResult, FactPushResult, UnresolvedEnvVarError, _build_certification_level_payload, @@ -43,7 +42,7 @@ def sample_gate_result() -> GateResult: mode=GateMode.BLOCK, message="Harbor evaluation passed", details={"engine": "harbor", "uplift": 0.15, "treatment_pass_rate": 0.9}, - evaluated_at=datetime(2026, 6, 21, 12, 0, 0, tzinfo=timezone.utc), + evaluated_at=datetime(2026, 6, 21, 12, 0, 0, tzinfo=UTC), ) @@ -228,7 +227,6 @@ def test_builds_fact_ref_from_prefix( sample_gate_result: GateResult, sample_push_facts_config: PushFactsConfig, ): - from abevalflow.compass_facts import FactPushResult with patch("abevalflow.compass_facts.push_gate_fact") as mock_push: mock_push.return_value = FactPushResult( @@ -286,9 +284,7 @@ def test_should_push_fact_returns_false_when_no_config(self): policy = GatePolicy() assert policy.should_push_fact("harbor") is False - def test_should_push_fact_returns_false_when_gate_not_configured( - self, sample_push_facts_config: PushFactsConfig - ): + def test_should_push_fact_returns_false_when_gate_not_configured(self, sample_push_facts_config: PushFactsConfig): from abevalflow.schemas import GatePolicy, GatePolicyItem policy = GatePolicy( @@ -297,9 +293,7 @@ def test_should_push_fact_returns_false_when_gate_not_configured( ) assert policy.should_push_fact("evaluation") is False - def test_should_push_fact_returns_true_when_configured( - self, sample_push_facts_config: PushFactsConfig - ): + def test_should_push_fact_returns_true_when_configured(self, sample_push_facts_config: PushFactsConfig): from abevalflow.schemas import GatePolicy, GatePolicyItem policy = GatePolicy( @@ -465,7 +459,7 @@ def test_no_failure_reason_when_check_failed(self): class TestPushCertificationFactsHierarchy: """Tests for push_certification_facts with pre-enforced hierarchy. - + Note: Hierarchy enforcement now happens in compute_certification(), not in push_certification_facts. These tests verify that push_certification_facts correctly passes through the pre-enforced .passed values. diff --git a/tests/test_db_models.py b/tests/test_db_models.py index 3b185cf..9f9dc20 100644 --- a/tests/test_db_models.py +++ b/tests/test_db_models.py @@ -3,7 +3,7 @@ from __future__ import annotations import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from unittest.mock import MagicMock import pytest @@ -75,9 +75,7 @@ def test_create_run(self, session: Session): session.commit() loaded = session.execute( - select(EvaluationRun).where( - EvaluationRun.pipeline_run_id == run.pipeline_run_id - ) + select(EvaluationRun).where(EvaluationRun.pipeline_run_id == run.pipeline_run_id) ).scalar_one() assert loaded.submission_name == "test-submission" assert loaded.recommendation == "pass" @@ -93,7 +91,7 @@ def test_created_at_default(self, session: Session): # Just verify the timestamp is recent (within last 60s). from datetime import timedelta - assert abs((datetime.now(timezone.utc).replace(tzinfo=None) - run.created_at.replace(tzinfo=None))) < timedelta(seconds=60) + assert abs(datetime.now(UTC).replace(tzinfo=None) - run.created_at.replace(tzinfo=None)) < timedelta(seconds=60) def test_unique_pipeline_run_id(self, session: Session): run_id = "duplicate-run" @@ -137,7 +135,7 @@ def test_repr(self): class TestTrial: def test_create_trial(self, session: Session): run = _make_run() - trial = Trial( + _trial = Trial( # Created via run relationship, loaded separately below run=run, variant="treatment", trial_name="trial-001", @@ -147,9 +145,7 @@ def test_create_trial(self, session: Session): session.add(run) session.commit() - loaded = session.execute( - select(Trial).where(Trial.trial_name == "trial-001") - ).scalar_one() + loaded = session.execute(select(Trial).where(Trial.trial_name == "trial-001")).scalar_one() assert loaded.variant == "treatment" assert loaded.reward == pytest.approx(0.85) assert loaded.passed is True @@ -190,8 +186,8 @@ def test_cascade_delete(self, session: Session): def test_relationship_back_populates(self, session: Session): run = _make_run() - t1 = Trial(run=run, variant="treatment", trial_name="t1", reward=0.9, passed=True) - t2 = Trial(run=run, variant="control", trial_name="c1", reward=0.0, passed=False) + _t1 = Trial(run=run, variant="treatment", trial_name="t1", reward=0.9, passed=True) + _t2 = Trial(run=run, variant="control", trial_name="c1", reward=0.0, passed=False) session.add(run) session.commit() @@ -199,9 +195,7 @@ def test_relationship_back_populates(self, session: Session): assert {t.trial_name for t in run.trials} == {"t1", "c1"} def test_repr(self): - t = Trial( - variant="treatment", trial_name="t-001", reward=0.75, passed=True - ) + t = Trial(variant="treatment", trial_name="t-001", reward=0.75, passed=True) r = repr(t) assert "t-001" in r assert "treatment" in r @@ -252,12 +246,8 @@ def _sample_result() -> AnalysisResult: submission_name="obs-test", provenance=Provenance(), summary=AnalysisSummary( - treatment=VariantSummary( - n_trials=10, n_passed=8, n_failed=2, pass_rate=0.8 - ), - control=VariantSummary( - n_trials=10, n_passed=5, n_failed=5, pass_rate=0.5 - ), + treatment=VariantSummary(n_trials=10, n_passed=8, n_failed=2, pass_rate=0.8), + control=VariantSummary(n_trials=10, n_passed=5, n_failed=5, pass_rate=0.5), uplift=0.3, recommendation=Recommendation.PASS, ), diff --git a/tests/test_engine_registry.py b/tests/test_engine_registry.py index 5244e20..b9266f0 100644 --- a/tests/test_engine_registry.py +++ b/tests/test_engine_registry.py @@ -1,8 +1,6 @@ """Tests for engine registry and adapters.""" import json -import tempfile -from pathlib import Path import pytest @@ -11,7 +9,7 @@ from abevalflow.engines.ase import ASEEngine from abevalflow.engines.harbor import HarborEngine from abevalflow.engines.mcpchecker import MCPCheckerEngine -from abevalflow.gates.base import GateMode, GateType +from abevalflow.gates.base import GateType from abevalflow.schemas import GatePolicy, GatePolicyItem diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 80fd4a5..6e56538 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -13,7 +13,6 @@ from abevalflow.schemas import ( CopySpec, ExperimentConfig, - ExperimentType, VariantSpec, ) @@ -79,7 +78,8 @@ def test_default_config_returns_skill_strategy(self) -> None: class TestSkillExperimentStrategy: def test_treatment_copy_specs_include_skills_and_docs( - self, submission_dir: Path, + self, + submission_dir: Path, ) -> None: config = ExperimentConfig() strategy = SkillExperimentStrategy(config) @@ -89,7 +89,8 @@ def test_treatment_copy_specs_include_skills_and_docs( assert "docs" in srcs def test_control_copy_specs_empty_by_default( - self, submission_dir: Path, + self, + submission_dir: Path, ) -> None: config = ExperimentConfig() strategy = SkillExperimentStrategy(config) @@ -97,7 +98,8 @@ def test_control_copy_specs_empty_by_default( assert specs == [] def test_treatment_skips_missing_dirs( - self, submission_dir_no_docs: Path, + self, + submission_dir_no_docs: Path, ) -> None: config = ExperimentConfig() strategy = SkillExperimentStrategy(config) @@ -136,7 +138,8 @@ def test_base_context_not_mutated(self, submission_dir: Path) -> None: assert "skills_dir" not in base def test_treatment_context_filters_missing_docs( - self, submission_dir_no_docs: Path, + self, + submission_dir_no_docs: Path, ) -> None: """copy_pairs must not include dirs that don't exist in submission.""" config = ExperimentConfig() diff --git a/tests/test_gate_registry.py b/tests/test_gate_registry.py index 96672f9..260815f 100644 --- a/tests/test_gate_registry.py +++ b/tests/test_gate_registry.py @@ -1,7 +1,6 @@ """Tests for security and quality gate registries.""" import json -from pathlib import Path import pytest @@ -47,9 +46,7 @@ class TestCiscoGate: """Tests for Cisco security gate.""" def test_evaluate_disabled(self, tmp_path): - policy = GatePolicy( - gates={"security": GatePolicyItem(mode=GateMode.DISABLED)} - ) + policy = GatePolicy(gates={"security": GatePolicyItem(mode=GateMode.DISABLED)}) gate = CiscoGate() result = gate.evaluate(tmp_path, policy) @@ -90,9 +87,7 @@ def test_evaluate_with_findings_warn_mode(self, tmp_path): } (tmp_path / "security-scan.json").write_text(json.dumps(scan_data)) - policy = GatePolicy( - gates={"security": GatePolicyItem(mode=GateMode.WARN)} - ) + policy = GatePolicy(gates={"security": GatePolicyItem(mode=GateMode.WARN)}) gate = CiscoGate() result = gate.evaluate(tmp_path, policy) @@ -108,9 +103,7 @@ def test_evaluate_with_high_findings_block_mode(self, tmp_path): } (tmp_path / "security-scan.json").write_text(json.dumps(scan_data)) - policy = GatePolicy( - gates={"security": GatePolicyItem(mode=GateMode.BLOCK)} - ) + policy = GatePolicy(gates={"security": GatePolicyItem(mode=GateMode.BLOCK)}) gate = CiscoGate() result = gate.evaluate(tmp_path, policy) @@ -125,9 +118,7 @@ def test_evaluate_with_critical_finding(self, tmp_path): } (tmp_path / "security-scan.json").write_text(json.dumps(scan_data)) - policy = GatePolicy( - gates={"security": GatePolicyItem(mode=GateMode.BLOCK)} - ) + policy = GatePolicy(gates={"security": GatePolicyItem(mode=GateMode.BLOCK)}) gate = CiscoGate() result = gate.evaluate(tmp_path, policy) @@ -143,9 +134,7 @@ def test_evaluate_only_low_findings_block_mode(self, tmp_path): } (tmp_path / "security-scan.json").write_text(json.dumps(scan_data)) - policy = GatePolicy( - gates={"security": GatePolicyItem(mode=GateMode.BLOCK)} - ) + policy = GatePolicy(gates={"security": GatePolicyItem(mode=GateMode.BLOCK)}) gate = CiscoGate() result = gate.evaluate(tmp_path, policy) @@ -178,9 +167,7 @@ class TestLLMReviewGate: """Tests for LLM review quality gate.""" def test_evaluate_disabled(self, tmp_path): - policy = GatePolicy( - gates={"quality": GatePolicyItem(mode=GateMode.DISABLED)} - ) + policy = GatePolicy(gates={"quality": GatePolicyItem(mode=GateMode.DISABLED)}) gate = LLMReviewGate() result = gate.evaluate(tmp_path, policy) @@ -259,9 +246,7 @@ def test_evaluate_warn_recommendation_block_mode(self, tmp_path): } (tmp_path / "_ai_review.json").write_text(json.dumps(review_data)) - policy = GatePolicy( - gates={"quality": GatePolicyItem(mode=GateMode.BLOCK, threshold=0.6)} - ) + policy = GatePolicy(gates={"quality": GatePolicyItem(mode=GateMode.BLOCK, threshold=0.6)}) gate = LLMReviewGate() result = gate.evaluate(tmp_path, policy) @@ -280,9 +265,7 @@ def test_evaluate_warn_recommendation_warn_mode(self, tmp_path): } (tmp_path / "_ai_review.json").write_text(json.dumps(review_data)) - policy = GatePolicy( - gates={"quality": GatePolicyItem(mode=GateMode.WARN)} - ) + policy = GatePolicy(gates={"quality": GatePolicyItem(mode=GateMode.WARN)}) gate = LLMReviewGate() result = gate.evaluate(tmp_path, policy) @@ -302,9 +285,7 @@ def test_block_mode_fail_recommendation_high_score(self, tmp_path): } (tmp_path / "_ai_review.json").write_text(json.dumps(review_data)) - policy = GatePolicy( - gates={"quality": GatePolicyItem(mode=GateMode.BLOCK, threshold=0.6)} - ) + policy = GatePolicy(gates={"quality": GatePolicyItem(mode=GateMode.BLOCK, threshold=0.6)}) gate = LLMReviewGate() result = gate.evaluate(tmp_path, policy) @@ -324,9 +305,7 @@ def test_block_mode_pass_recommendation_low_score(self, tmp_path): } (tmp_path / "_ai_review.json").write_text(json.dumps(review_data)) - policy = GatePolicy( - gates={"quality": GatePolicyItem(mode=GateMode.BLOCK, threshold=0.6)} - ) + policy = GatePolicy(gates={"quality": GatePolicyItem(mode=GateMode.BLOCK, threshold=0.6)}) gate = LLMReviewGate() result = gate.evaluate(tmp_path, policy) @@ -336,9 +315,7 @@ def test_block_mode_pass_recommendation_low_score(self, tmp_path): def test_block_mode_missing_artifact_fails(self, tmp_path): """In BLOCK mode, missing _ai_review.json fails the gate.""" - policy = GatePolicy( - gates={"quality": GatePolicyItem(mode=GateMode.BLOCK)} - ) + policy = GatePolicy(gates={"quality": GatePolicyItem(mode=GateMode.BLOCK)}) gate = LLMReviewGate() result = gate.evaluate(tmp_path, policy) @@ -351,9 +328,7 @@ class TestCiscoGateBlockMode: def test_block_mode_missing_artifact_fails(self, tmp_path): """In BLOCK mode, missing security-scan.json fails the gate.""" - policy = GatePolicy( - gates={"security": GatePolicyItem(mode=GateMode.BLOCK)} - ) + policy = GatePolicy(gates={"security": GatePolicyItem(mode=GateMode.BLOCK)}) gate = CiscoGate() result = gate.evaluate(tmp_path, policy) @@ -362,9 +337,7 @@ def test_block_mode_missing_artifact_fails(self, tmp_path): def test_warn_mode_missing_artifact_passes(self, tmp_path): """In WARN mode, missing security-scan.json passes the gate.""" - policy = GatePolicy( - gates={"security": GatePolicyItem(mode=GateMode.WARN)} - ) + policy = GatePolicy(gates={"security": GatePolicyItem(mode=GateMode.WARN)}) gate = CiscoGate() result = gate.evaluate(tmp_path, policy) diff --git a/tests/test_generate_eval_config.py b/tests/test_generate_eval_config.py index db6390a..7c5d780 100644 --- a/tests/test_generate_eval_config.py +++ b/tests/test_generate_eval_config.py @@ -74,8 +74,12 @@ class TestBuildVariantConfigPrebuilt: def test_basic_structure(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, ) assert config["job_name"] == "my-submission-treatment" assert config["n_attempts"] == 20 @@ -87,16 +91,24 @@ def test_basic_structure(self, minimal_submission: Path): def test_image_ref_in_env_kwargs(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, ) assert config["environment"]["kwargs"]["image_ref"] == TREATMENT_REF def test_control_variant_naming(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "control", CONTROL_DIR, "prebuilt", - jobs_dir="results/control", image_ref=CONTROL_REF, + meta, + "control", + CONTROL_DIR, + "prebuilt", + jobs_dir="results/control", + image_ref=CONTROL_REF, ) assert config["job_name"] == "my-submission-control" assert config["tasks"][0]["path"] == CONTROL_DIR @@ -104,8 +116,12 @@ def test_control_variant_naming(self, minimal_submission: Path): def test_no_force_build(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, ) assert "force_build" not in config["environment"] @@ -113,8 +129,12 @@ def test_missing_image_ref_raises(self, minimal_submission: Path): meta = load_metadata(minimal_submission) with pytest.raises(ValueError, match="image_ref is required"): build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref="", + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref="", ) @@ -122,7 +142,10 @@ class TestBuildVariantConfigLocalBuild: def test_kwargs_without_image_ref(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "local-build", + meta, + "treatment", + TREATMENT_DIR, + "local-build", jobs_dir="results/treatment", ) kwargs = config["environment"]["kwargs"] @@ -133,7 +156,10 @@ def test_kwargs_without_image_ref(self, minimal_submission: Path): def test_force_build_enabled(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "local-build", + meta, + "treatment", + TREATMENT_DIR, + "local-build", jobs_dir="results/treatment", ) assert config["environment"]["force_build"] is True @@ -143,7 +169,10 @@ class TestCustomMetadataFields: def test_n_trials_from_metadata(self, custom_submission: Path): meta = load_metadata(custom_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "local-build", + meta, + "treatment", + TREATMENT_DIR, + "local-build", jobs_dir="results/treatment", ) assert config["n_attempts"] == 10 @@ -151,7 +180,10 @@ def test_n_trials_from_metadata(self, custom_submission: Path): def test_resource_overrides(self, custom_submission: Path): meta = load_metadata(custom_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "local-build", + meta, + "treatment", + TREATMENT_DIR, + "local-build", jobs_dir="results/treatment", ) assert config["environment"]["override_memory_mb"] == 4096 @@ -160,7 +192,10 @@ def test_resource_overrides(self, custom_submission: Path): def test_timeout_multipliers(self, custom_submission: Path): meta = load_metadata(custom_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "local-build", + meta, + "treatment", + TREATMENT_DIR, + "local-build", jobs_dir="results/treatment", ) assert config["agent_timeout_multiplier"] == pytest.approx(2.0) @@ -171,7 +206,10 @@ def test_timeout_multipliers(self, custom_submission: Path): def test_default_timeouts_produce_1x_multiplier(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "local-build", + meta, + "treatment", + TREATMENT_DIR, + "local-build", jobs_dir="results/treatment", ) assert config["agent_timeout_multiplier"] == pytest.approx(1.0) @@ -182,7 +220,10 @@ def test_default_timeouts_produce_1x_multiplier(self, minimal_submission: Path): def test_custom_jobs_dir(self, minimal_submission: Path): meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "local-build", + meta, + "treatment", + TREATMENT_DIR, + "local-build", jobs_dir="/workspace/results/treatment", ) assert config["jobs_dir"] == "/workspace/results/treatment" @@ -193,8 +234,12 @@ def test_default_oracle_when_no_llm(self, minimal_submission: Path): """No LLM params at all -> oracle agent (empty dict).""" meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, ) assert config["agents"] == [{}] @@ -202,8 +247,12 @@ def test_claude_code_default(self, minimal_submission: Path): """Model without wrapper -> claude-code agent with Anthropic env.""" meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_model="claude-sonnet", llm_api_base="http://litellm:4000", llm_api_key="mock", @@ -218,8 +267,12 @@ def test_claude_code_model_only(self, minimal_submission: Path): """Model without api_base or key -> claude-code, model_name, no env.""" meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_model="claude-sonnet", ) agent = config["agents"][0] @@ -231,8 +284,12 @@ def test_wrapper_agent_opencode(self, minimal_submission: Path): """Wrapper agent -> opencode with OPENAI_BASE_URL.""" meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_agent_wrapper="opencode", llm_model="openai/llama3", llm_api_base="http://litellm:4000", @@ -246,8 +303,12 @@ def test_wrapper_without_api_base(self, minimal_submission: Path): """Wrapper agent with model but no api_base -> no env block.""" meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_agent_wrapper="opencode", llm_model="openai/gpt-4o", ) @@ -257,7 +318,9 @@ def test_wrapper_without_api_base(self, minimal_submission: Path): assert "env" not in agent def test_llm_config_in_generated_yaml( - self, minimal_submission: Path, tmp_path: Path, + self, + minimal_submission: Path, + tmp_path: Path, ): out_dir = tmp_path / "configs" configs = generate_eval_configs( @@ -277,9 +340,7 @@ def test_llm_config_in_generated_yaml( agent = configs[variant]["agents"][0] assert agent["name"] == "claude-code" assert agent["model_name"] == "claude-sonnet" - loaded = yaml.safe_load( - (out_dir / f"{variant}-config.yaml").read_text() - ) + loaded = yaml.safe_load((out_dir / f"{variant}-config.yaml").read_text()) assert loaded["agents"][0]["name"] == "claude-code" assert loaded["agents"][0]["model_name"] == "claude-sonnet" @@ -318,8 +379,12 @@ def test_wrapper_override(self, submission_with_wrapper_override: Path): """Metadata switches from claude-code default to opencode wrapper.""" meta = load_metadata(submission_with_wrapper_override) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_model="claude-sonnet", llm_api_base="http://litellm:4000", llm_api_key="mock", @@ -333,8 +398,12 @@ def test_model_override_keeps_claude_code(self, submission_model_override: Path) """Overriding only model keeps claude-code as the agent.""" meta = load_metadata(submission_model_override) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_model="claude-sonnet", llm_api_base="http://litellm:4000", llm_api_key="mock", @@ -349,8 +418,12 @@ def test_no_llm_block_uses_pipeline_defaults(self, minimal_submission: Path): """No llm: in metadata -> pipeline defaults (claude-code) used.""" meta = load_metadata(minimal_submission) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_model="claude-sonnet", llm_api_base="http://litellm:4000", llm_api_key="mock", @@ -370,8 +443,12 @@ def test_oracle_override(self, tmp_path: Path): (sub / "metadata.yaml").write_text(yaml.dump(meta)) meta = load_metadata(sub) config = build_variant_config( - meta, "treatment", TREATMENT_DIR, "prebuilt", - jobs_dir="results/treatment", image_ref=TREATMENT_REF, + meta, + "treatment", + TREATMENT_DIR, + "prebuilt", + jobs_dir="results/treatment", + image_ref=TREATMENT_REF, llm_model="claude-sonnet", llm_api_base="http://litellm:4000", llm_api_key="mock", @@ -399,7 +476,9 @@ def test_writes_two_yaml_files(self, minimal_submission: Path, tmp_path: Path): assert "control" in configs def test_variant_jobs_dirs_are_separate( - self, minimal_submission: Path, tmp_path: Path, + self, + minimal_submission: Path, + tmp_path: Path, ): out_dir = tmp_path / "configs" configs = generate_eval_configs( @@ -416,7 +495,9 @@ def test_variant_jobs_dirs_are_separate( assert configs["control"]["jobs_dir"] == "eval-results/control" def test_each_config_has_single_task( - self, minimal_submission: Path, tmp_path: Path, + self, + minimal_submission: Path, + tmp_path: Path, ): out_dir = tmp_path / "configs" configs = generate_eval_configs( @@ -445,9 +526,7 @@ def test_yaml_roundtrips(self, minimal_submission: Path, tmp_path: Path): control_image_ref=CONTROL_REF, ) for variant in ("treatment", "control"): - loaded = yaml.safe_load( - (out_dir / f"{variant}-config.yaml").read_text() - ) + loaded = yaml.safe_load((out_dir / f"{variant}-config.yaml").read_text()) assert loaded["job_name"] == configs[variant]["job_name"] assert loaded["n_attempts"] == configs[variant]["n_attempts"] @@ -467,54 +546,86 @@ def test_creates_output_dir(self, minimal_submission: Path, tmp_path: Path): class TestMainCLI: def test_prebuilt_mode(self, minimal_submission: Path, tmp_path: Path): out_dir = tmp_path / "out" - rc = main([ - "--submission-dir", str(minimal_submission), - "--treatment-task-dir", TREATMENT_DIR, - "--control-task-dir", CONTROL_DIR, - "--output-dir", str(out_dir), - "--eval-mode", "prebuilt", - "--treatment-image-ref", TREATMENT_REF, - "--control-image-ref", CONTROL_REF, - ]) + rc = main( + [ + "--submission-dir", + str(minimal_submission), + "--treatment-task-dir", + TREATMENT_DIR, + "--control-task-dir", + CONTROL_DIR, + "--output-dir", + str(out_dir), + "--eval-mode", + "prebuilt", + "--treatment-image-ref", + TREATMENT_REF, + "--control-image-ref", + CONTROL_REF, + ] + ) assert rc == 0 t_config = yaml.safe_load((out_dir / "treatment-config.yaml").read_text()) assert t_config["environment"]["kwargs"]["image_ref"] == TREATMENT_REF def test_local_build_mode(self, minimal_submission: Path, tmp_path: Path): out_dir = tmp_path / "out" - rc = main([ - "--submission-dir", str(minimal_submission), - "--treatment-task-dir", TREATMENT_DIR, - "--control-task-dir", CONTROL_DIR, - "--output-dir", str(out_dir), - "--eval-mode", "local-build", - ]) + rc = main( + [ + "--submission-dir", + str(minimal_submission), + "--treatment-task-dir", + TREATMENT_DIR, + "--control-task-dir", + CONTROL_DIR, + "--output-dir", + str(out_dir), + "--eval-mode", + "local-build", + ] + ) assert rc == 0 t_config = yaml.safe_load((out_dir / "treatment-config.yaml").read_text()) assert "image_ref" not in t_config["environment"].get("kwargs", {}) assert t_config["environment"]["kwargs"]["memory_limit_multiplier"] == 1.5 def test_prebuilt_missing_refs_exits_error( - self, minimal_submission: Path, tmp_path: Path, + self, + minimal_submission: Path, + tmp_path: Path, ): out_dir = tmp_path / "out" with pytest.raises(SystemExit) as exc_info: - main([ - "--submission-dir", str(minimal_submission), - "--treatment-task-dir", TREATMENT_DIR, - "--control-task-dir", CONTROL_DIR, - "--output-dir", str(out_dir), - "--eval-mode", "prebuilt", - ]) + main( + [ + "--submission-dir", + str(minimal_submission), + "--treatment-task-dir", + TREATMENT_DIR, + "--control-task-dir", + CONTROL_DIR, + "--output-dir", + str(out_dir), + "--eval-mode", + "prebuilt", + ] + ) assert exc_info.value.code == 2 def test_nonexistent_submission_dir(self, tmp_path: Path): out_dir = tmp_path / "out" - rc = main([ - "--submission-dir", str(tmp_path / "no-such-dir"), - "--treatment-task-dir", TREATMENT_DIR, - "--control-task-dir", CONTROL_DIR, - "--output-dir", str(out_dir), - "--eval-mode", "local-build", - ]) + rc = main( + [ + "--submission-dir", + str(tmp_path / "no-such-dir"), + "--treatment-task-dir", + TREATMENT_DIR, + "--control-task-dir", + CONTROL_DIR, + "--output-dir", + str(out_dir), + "--eval-mode", + "local-build", + ] + ) assert rc == 1 diff --git a/tests/test_generate_tests.py b/tests/test_generate_tests.py index 6dbeae5..1536ffe 100644 --- a/tests/test_generate_tests.py +++ b/tests/test_generate_tests.py @@ -10,7 +10,6 @@ import yaml from scripts.generate_tests import ( - DEFAULT_MAX_RETRIES, _correction_pass, generate, main, @@ -36,11 +35,13 @@ - Verify error handling """ -MOCK_ANALYSIS = json.dumps({ - "novel_aspects": ["Check for anti-patterns specific to this project"], - "common_knowledge": ["Basic code review practices"], - "test_focus_areas": ["Verify anti-pattern detection covers project-specific cases"], -}) +MOCK_ANALYSIS = json.dumps( + { + "novel_aspects": ["Check for anti-patterns specific to this project"], + "common_knowledge": ["Basic code review practices"], + "test_focus_areas": ["Verify anti-pattern detection covers project-specific cases"], + } +) MOCK_INSTRUCTION = ( "# Code Review Task\n\n" @@ -62,10 +63,12 @@ " assert isinstance(result, list)\n" ) -MOCK_SCENARIO_BRIEF = json.dumps({ - "project_files": {"review.py": "Main module"}, - "expected_outputs": {"review_result": "list of issues"}, -}) +MOCK_SCENARIO_BRIEF = json.dumps( + { + "project_files": {"review.py": "Main module"}, + "expected_outputs": {"review_result": "list of issues"}, + } +) MOCK_JUDGE_SKIP = "SKIP" MOCK_JUDGE_CODE = "score = 0.9\n" @@ -77,6 +80,7 @@ "reviewer_results": {}, } + def _five_step_responses( analysis: str = MOCK_ANALYSIS, scenario_brief: str = MOCK_SCENARIO_BRIEF, @@ -248,9 +252,7 @@ def test_quality_criteria_included_when_skill_fetched( ) -> None: skill_cache = tmp_path / "_skill_cache" / "agentic-contribution-skill" skill_cache.mkdir(parents=True) - (skill_cache / "SKILL.md").write_text( - "---\nname: test\n---\n\n## Workflow\n\nDo good work.\n" - ) + (skill_cache / "SKILL.md").write_text("---\nname: test\n---\n\n## Workflow\n\nDo good work.\n") mock_fetch.return_value = skill_cache / "SKILL.md" mock_chat.side_effect = _five_step_responses() @@ -596,9 +598,7 @@ def test_feedback_injected_on_retry( retry_instruction_call = mock_chat.call_args_list[5] messages = ( - retry_instruction_call.args[0] - if retry_instruction_call.args - else retry_instruction_call.kwargs["messages"] + retry_instruction_call.args[0] if retry_instruction_call.args else retry_instruction_call.kwargs["messages"] ) user_msg = messages[1]["content"] assert "Previous Attempt Issues" in user_msg @@ -724,12 +724,8 @@ def oracle_submission(self, tmp_path: Path) -> Path: oracle = sub / "oracle" oracle.mkdir() (oracle / "instruction.md").write_text("\n# Task\nDo it.\n") - (oracle / "test_outputs.py").write_text( - "# #ai-generated-oracle\ndef test_ok(): assert True\n" - ) - (oracle / "llm_judge.py").write_text( - "# #ai-generated-oracle\nscore = 0.9\n" - ) + (oracle / "test_outputs.py").write_text("# #ai-generated-oracle\ndef test_ok(): assert True\n") + (oracle / "llm_judge.py").write_text("# #ai-generated-oracle\nscore = 0.9\n") return sub @patch("scripts.generate_tests.pytest_collect_check", return_value=[]) diff --git a/tests/test_generation_validator.py b/tests/test_generation_validator.py index 9cf59f9..3561e47 100644 --- a/tests/test_generation_validator.py +++ b/tests/test_generation_validator.py @@ -93,10 +93,12 @@ def test_pass_response(self, mock_chat: MagicMock, submission: Path) -> None: @patch("abevalflow.generation_validator.llm_client.chat_completion") def test_fail_response(self, mock_chat: MagicMock, submission: Path) -> None: - mock_chat.return_value = json.dumps({ - "pass": False, - "issues": ["instruction does not match skill"], - }) + mock_chat.return_value = json.dumps( + { + "pass": False, + "issues": ["instruction does not match skill"], + } + ) result = content_check(submission) assert result["passed"] is False assert "instruction does not match skill" in result["issues"] @@ -151,6 +153,7 @@ def test_no_tests_collected_returns_error(self, submission: Path) -> None: @patch("abevalflow.generation_validator.subprocess.run") def test_timeout_returns_error(self, mock_run: MagicMock, submission: Path) -> None: import subprocess + mock_run.side_effect = subprocess.TimeoutExpired(cmd="pytest", timeout=10) errors = pytest_collect_check(submission) assert any("timed out" in e for e in errors) @@ -228,10 +231,12 @@ def test_pass_response(self, mock_chat: MagicMock, submission: Path) -> None: @patch("abevalflow.generation_validator.llm_client.chat_completion") def test_fail_response(self, mock_chat: MagicMock, submission: Path) -> None: - mock_chat.return_value = json.dumps({ - "pass": False, - "issues": ["tests are not deterministic"], - }) + mock_chat.return_value = json.dumps( + { + "pass": False, + "issues": ["tests are not deterministic"], + } + ) result = final_review(submission) assert result["passed"] is False assert "tests are not deterministic" in result["issues"] diff --git a/tests/test_mcpchecker_integration.py b/tests/test_mcpchecker_integration.py index f968c51..241b5cf 100644 --- a/tests/test_mcpchecker_integration.py +++ b/tests/test_mcpchecker_integration.py @@ -1,7 +1,6 @@ """Tests for MCPChecker integration (validation, aggregation, storage).""" import json -import tempfile from pathlib import Path import pytest @@ -11,7 +10,6 @@ MCPCheckerResult, MCPCheckerTaskResult, ) -from abevalflow.report import Provenance from abevalflow.schemas import EvalEngine from scripts.aggregate_mcpchecker import aggregate_mcpchecker_results, extract_task_results from scripts.validate import validate_submission @@ -92,6 +90,7 @@ def test_missing_mcp_config(self, mcpchecker_submission_dir: Path): def test_missing_tasks_dir(self, mcpchecker_submission_dir: Path): """Missing tasks/ directory should fail validation.""" import shutil + shutil.rmtree(mcpchecker_submission_dir / "tasks") errors = validate_submission(mcpchecker_submission_dir, EvalEngine.MCPCHECKER) assert any("tasks/ directory is required" in e for e in errors) @@ -137,11 +136,14 @@ def sample_mcpchecker_output(self) -> dict: "taskId": "task-1", "taskName": "Health Check", "status": "passed", - "toolCalls": [ - {"server": "test-server", "tool": "health", "success": True} - ], + "toolCalls": [{"server": "test-server", "tool": "health", "success": True}], "llmJudgeResults": [ - {"type": "contains", "expected": "healthy", "passed": True, "reason": "Output contains 'healthy'"} + { + "type": "contains", + "expected": "healthy", + "passed": True, + "reason": "Output contains 'healthy'", + } ], "durationMs": 1500, }, @@ -149,9 +151,7 @@ def sample_mcpchecker_output(self) -> dict: "taskId": "task-2", "taskName": "Data Query", "status": "failed", - "toolCalls": [ - {"server": "test-server", "tool": "query", "success": False} - ], + "toolCalls": [{"server": "test-server", "tool": "query", "success": False}], "llmJudgeResults": [ {"type": "exact", "expected": "success", "passed": False, "reason": "Output does not match"} ], @@ -206,12 +206,9 @@ def test_overall_score_calculation(self, tmp_path: Path): output = { "evalName": "score-test", "taskResults": [ - {"taskId": f"t{i}", "taskName": f"Task {i}", "status": "passed", "toolCalls": []} - for i in range(7) - ] + [ - {"taskId": f"t{i}", "taskName": f"Task {i}", "status": "failed", "toolCalls": []} - for i in range(7, 10) + {"taskId": f"t{i}", "taskName": f"Task {i}", "status": "passed", "toolCalls": []} for i in range(7) ] + + [{"taskId": f"t{i}", "taskName": f"Task {i}", "status": "failed", "toolCalls": []} for i in range(7, 10)], } output_file = tmp_path / "out.json" output_file.write_text(json.dumps(output)) @@ -230,7 +227,7 @@ def test_recommendation_threshold(self, tmp_path: Path): "taskResults": [ {"taskId": str(i), "taskName": f"Task {i}", "status": "passed" if i < 7 else "failed", "toolCalls": []} for i in range(10) - ] + ], } output_file = tmp_path / "pass.json" output_file.write_text(json.dumps(output_pass)) @@ -243,7 +240,7 @@ def test_recommendation_threshold(self, tmp_path: Path): "taskResults": [ {"taskId": str(i), "taskName": f"Task {i}", "status": "passed" if i < 6 else "failed", "toolCalls": []} for i in range(10) - ] + ], } output_file = tmp_path / "fail.json" output_file.write_text(json.dumps(output_fail)) diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 1e0ea43..0e6a07a 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta import pytest from sqlalchemy import create_engine @@ -74,7 +74,7 @@ def test_no_runs_returns_not_degraded(self, engine): def test_single_run_returns_not_degraded(self, engine, session): """When only one run exists, should return not degraded.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 0.8, now) result = check_degradation(engine, "test-skill") @@ -87,7 +87,7 @@ def test_single_run_returns_not_degraded(self, engine, session): def test_healthy_no_degradation(self, engine, session): """When current score is similar to previous, no degradation.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 0.80, now - timedelta(days=1)) _create_run(session, "test-skill", "run-002", 0.82, now) @@ -101,7 +101,7 @@ def test_healthy_no_degradation(self, engine, session): def test_degradation_detected(self, engine, session): """When score drops significantly, degradation is detected.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 0.90, now - timedelta(days=1)) _create_run(session, "test-skill", "run-002", 0.70, now) @@ -117,7 +117,7 @@ def test_degradation_detected(self, engine, session): def test_exact_threshold_not_degraded(self, engine, session): """When ratio equals threshold exactly, no degradation.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 1.0, now - timedelta(days=1)) _create_run(session, "test-skill", "run-002", 0.85, now) @@ -128,7 +128,7 @@ def test_exact_threshold_not_degraded(self, engine, session): def test_just_below_threshold_is_degraded(self, engine, session): """When ratio is just below threshold, degradation is detected.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 1.0, now - timedelta(days=1)) _create_run(session, "test-skill", "run-002", 0.84, now) @@ -139,7 +139,7 @@ def test_just_below_threshold_is_degraded(self, engine, session): def test_previous_score_zero(self, engine, session): """Edge case: previous score is zero.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 0.0, now - timedelta(days=1)) _create_run(session, "test-skill", "run-002", 0.5, now) @@ -150,7 +150,7 @@ def test_previous_score_zero(self, engine, session): def test_both_scores_zero(self, engine, session): """Edge case: both scores are zero.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 0.0, now - timedelta(days=1)) _create_run(session, "test-skill", "run-002", 0.0, now) @@ -161,7 +161,7 @@ def test_both_scores_zero(self, engine, session): def test_filters_by_submission_name(self, engine, session): """Should only consider runs for the specified submission.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "skill-a", "run-a1", 0.9, now - timedelta(days=2)) _create_run(session, "skill-a", "run-a2", 0.85, now - timedelta(days=1)) _create_run(session, "skill-b", "run-b1", 0.5, now) @@ -175,7 +175,7 @@ def test_filters_by_submission_name(self, engine, session): def test_custom_threshold(self, engine, session): """Custom threshold should be respected.""" - now = datetime.now(timezone.utc) + now = datetime.now(UTC) _create_run(session, "test-skill", "run-001", 1.0, now - timedelta(days=1)) _create_run(session, "test-skill", "run-002", 0.75, now) diff --git a/tests/test_publish.py b/tests/test_publish.py index ac87beb..290c4c7 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from datetime import UTC from pathlib import Path from unittest.mock import MagicMock, patch @@ -66,9 +67,9 @@ def test_format(self): assert "abevalflow-xyz" in prefix def test_datestamp_is_today(self): - from datetime import datetime, timezone + from datetime import datetime - today = datetime.now(timezone.utc).strftime("%Y%m%d") + today = datetime.now(UTC).strftime("%Y%m%d") prefix = _build_artifact_prefix("test", "run1") assert prefix.startswith(today) @@ -137,14 +138,10 @@ def workspace_with_generated(self, tmp_path: Path) -> Path: """Workspace with generated files under submissions//.""" sub_dir = tmp_path / "submissions" / "ai-skill" sub_dir.mkdir(parents=True) - (sub_dir / "instruction.md").write_text( - "\n\n# Task\nDo something.\n" - ) + (sub_dir / "instruction.md").write_text("\n\n# Task\nDo something.\n") tests = sub_dir / "tests" tests.mkdir() - (tests / "test_outputs.py").write_text( - "# Generated by AI\ndef test_ok(): assert True\n" - ) + (tests / "test_outputs.py").write_text("# Generated by AI\ndef test_ok(): assert True\n") (tests / "llm_judge.py").write_text("# Generated by AI\nscore = 0.9\n") (sub_dir / "scenario_brief.json").write_text("{}") return tmp_path @@ -427,9 +424,7 @@ def test_uploads_agent_and_verifier_files(self, mock_minio_cls, results_dir): ) assert count == 8 # 4 files x 2 variants (agent.log, solution.py, verifier.log, result.json) - uploaded_names = [ - call[0][1] for call in mock_client.fput_object.call_args_list - ] + uploaded_names = [call[0][1] for call in mock_client.fput_object.call_args_list] assert any("agent/agent.log" in n for n in uploaded_names) assert any("verifier/verifier.log" in n for n in uploaded_names) assert all(n.startswith("20260428_test_run1/debug/") for n in uploaded_names) @@ -452,9 +447,7 @@ def test_skips_unrelated_files_but_uploads_trial_root_files(self, mock_minio_cls secret_key="secret", ) - uploaded_names = [ - call[0][1] for call in mock_client.fput_object.call_args_list - ] + uploaded_names = [call[0][1] for call in mock_client.fput_object.call_args_list] assert not any("random_file.txt" in n for n in uploaded_names) assert any("result.json" in n for n in uploaded_names) diff --git a/tests/test_query_results.py b/tests/test_query_results.py index 80d09ff..1997b40 100644 --- a/tests/test_query_results.py +++ b/tests/test_query_results.py @@ -3,7 +3,7 @@ from __future__ import annotations import uuid -from datetime import datetime, timezone, timedelta +from datetime import UTC, datetime, timedelta import pytest from sqlalchemy import create_engine @@ -56,7 +56,7 @@ def session_factory(): @pytest.fixture() def seeded_factory(session_factory): """Seed 3 runs: 2 for 'alpha', 1 for 'beta'.""" - now = datetime.now(timezone.utc).replace(tzinfo=None) + now = datetime.now(UTC).replace(tzinfo=None) with session_factory() as session: session.add( _make_run( diff --git a/tests/test_scaffold.py b/tests/test_scaffold.py index fc0ef5d..5b37a8d 100644 --- a/tests/test_scaffold.py +++ b/tests/test_scaffold.py @@ -62,13 +62,9 @@ def full_submission(valid_submission: Path) -> Path: supportive_docs.mkdir() (supportive_docs / "guide.md").write_text("# Guide\nOperational knowledge.\n") - (valid_submission / "tests" / "llm_judge.py").write_text( - "def judge(): return {'score': 1.0, 'rationale': 'ok'}\n" - ) + (valid_submission / "tests" / "llm_judge.py").write_text("def judge(): return {'score': 1.0, 'rationale': 'ok'}\n") - (valid_submission / "CLAUDE.md").write_text( - "# System Prompt\nYou are an SRE assistant.\n" - ) + (valid_submission / "CLAUDE.md").write_text("# System Prompt\nYou are an SRE assistant.\n") return valid_submission @@ -166,9 +162,7 @@ def test_task_toml_metadata_fields(self, valid_submission: Path, tmp_path: Path) assert '"test"' in content assert '"demo"' in content - def test_task_toml_is_valid_toml_without_llm_judge( - self, valid_submission: Path, tmp_path: Path - ): + def test_task_toml_is_valid_toml_without_llm_judge(self, valid_submission: Path, tmp_path: Path): output = tmp_path / "output" treatment, control = scaffold_submission(valid_submission, output, TEMPLATES_DIR) for d in (treatment, control): @@ -238,9 +232,7 @@ def test_docs_copied_only_for_treatment(self, full_submission: Path, tmp_path: P assert (treatment / "environment" / "docs" / "reference.md").is_file() assert not (control / "environment" / "docs").exists() - def test_dockerfile_includes_supportive_when_present( - self, full_submission: Path, tmp_path: Path - ): + def test_dockerfile_includes_supportive_when_present(self, full_submission: Path, tmp_path: Path): output = tmp_path / "output" treatment, _ = scaffold_submission(full_submission, output, TEMPLATES_DIR) content = (treatment / "environment" / "Dockerfile").read_text() @@ -432,6 +424,7 @@ def test_n_trials_accessible_from_metadata(self, valid_submission: Path, tmp_pat meta_path.write_text(yaml.dump(meta)) from abevalflow.schemas import SubmissionMetadata + parsed = SubmissionMetadata(**yaml.safe_load(meta_path.read_text())) assert parsed.experiment.n_trials == 42 @@ -457,7 +450,9 @@ def test_prompt_experiment_identical_to_skill(self, valid_submission: Path, tmp_ assert skill_ctrl == prompt_ctrl def test_missing_configured_dir_not_copied( - self, valid_submission: Path, tmp_path: Path, + self, + valid_submission: Path, + tmp_path: Path, ): """Configured copy dirs that don't exist in submission must not be copied.""" meta_path = valid_submission / "metadata.yaml" @@ -481,7 +476,9 @@ def test_missing_configured_dir_not_copied( assert not (treatment / "environment" / "nonexistent").exists() def test_control_with_skills_treatment_without( - self, valid_submission: Path, tmp_path: Path, + self, + valid_submission: Path, + tmp_path: Path, ): """Asymmetric config: control gets skills, treatment does not.""" meta_path = valid_submission / "metadata.yaml" @@ -507,7 +504,10 @@ class TestNonClaudeSkillWarning: """Verify the loud warning when skills_dir is used with a non-Claude agent.""" def test_non_claude_agent_with_skills_warns( - self, valid_submission: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture, + self, + valid_submission: Path, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, ): meta_path = valid_submission / "metadata.yaml" meta = yaml.safe_load(meta_path.read_text()) @@ -516,6 +516,7 @@ def test_non_claude_agent_with_skills_warns( output = tmp_path / "output" import logging + with caplog.at_level(logging.WARNING): scaffold_submission(valid_submission, output, TEMPLATES_DIR) @@ -523,17 +524,24 @@ def test_non_claude_agent_with_skills_warns( assert any("OPENCODE" in msg for msg in caplog.messages) def test_claude_agent_no_warning( - self, valid_submission: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture, + self, + valid_submission: Path, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, ): output = tmp_path / "output" import logging + with caplog.at_level(logging.WARNING): scaffold_submission(valid_submission, output, TEMPLATES_DIR) assert not any("NON-CLAUDE AGENT WRAPPER" in msg for msg in caplog.messages) def test_non_claude_without_skills_no_warning( - self, valid_submission: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture, + self, + valid_submission: Path, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, ): """Non-Claude agent with no skills_dir should not warn.""" meta_path = valid_submission / "metadata.yaml" @@ -548,6 +556,7 @@ def test_non_claude_without_skills_no_warning( output = tmp_path / "output" import logging + with caplog.at_level(logging.WARNING): scaffold_submission(valid_submission, output, TEMPLATES_DIR) diff --git a/tests/test_scorecard.py b/tests/test_scorecard.py index 7d2991b..59c0f83 100644 --- a/tests/test_scorecard.py +++ b/tests/test_scorecard.py @@ -1,9 +1,7 @@ """Tests for unified scorecard schemas and aggregation logic.""" import json -import tempfile -from datetime import datetime, timezone -from pathlib import Path +from datetime import UTC, datetime import pytest @@ -120,8 +118,22 @@ class TestCombinationLogic: def test_all_pass_success(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.9, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.SECURITY, gate_name="security", policy_key="cisco", passed=True, score=1.0, mode=GateMode.BLOCK), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.9, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.SECURITY, + gate_name="security", + policy_key="cisco", + passed=True, + score=1.0, + mode=GateMode.BLOCK, + ), ] policy = GatePolicy(combination=CombinationMode.ALL_PASS) rec, reason = apply_combination_logic(gates, policy) @@ -130,8 +142,22 @@ def test_all_pass_success(self): def test_all_pass_failure(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.9, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.SECURITY, gate_name="security", policy_key="cisco", passed=False, score=0.3, mode=GateMode.BLOCK), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.9, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.SECURITY, + gate_name="security", + policy_key="cisco", + passed=False, + score=0.3, + mode=GateMode.BLOCK, + ), ] policy = GatePolicy(combination=CombinationMode.ALL_PASS) rec, reason = apply_combination_logic(gates, policy) @@ -140,8 +166,22 @@ def test_all_pass_failure(self): def test_all_pass_warn_only(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.9, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.QUALITY, gate_name="quality", policy_key="llm-review", passed=False, score=0.4, mode=GateMode.WARN), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.9, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.QUALITY, + gate_name="quality", + policy_key="llm-review", + passed=False, + score=0.4, + mode=GateMode.WARN, + ), ] policy = GatePolicy(combination=CombinationMode.ALL_PASS) rec, reason = apply_combination_logic(gates, policy) @@ -150,8 +190,22 @@ def test_all_pass_warn_only(self): def test_any_pass_success(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.9, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="ase", passed=False, score=0.3, mode=GateMode.BLOCK), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.9, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="ase", + passed=False, + score=0.3, + mode=GateMode.BLOCK, + ), ] policy = GatePolicy(combination=CombinationMode.ANY_PASS) rec, reason = apply_combination_logic(gates, policy) @@ -159,8 +213,22 @@ def test_any_pass_success(self): def test_any_pass_failure(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=False, score=0.3, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="ase", passed=False, score=0.2, mode=GateMode.BLOCK), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=False, + score=0.3, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="ase", + passed=False, + score=0.2, + mode=GateMode.BLOCK, + ), ] policy = GatePolicy(combination=CombinationMode.ANY_PASS) rec, reason = apply_combination_logic(gates, policy) @@ -168,8 +236,22 @@ def test_any_pass_failure(self): def test_weighted_pass(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.9, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="ase", passed=True, score=0.6, mode=GateMode.BLOCK), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.9, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="ase", + passed=True, + score=0.6, + mode=GateMode.BLOCK, + ), ] policy = GatePolicy( combination=CombinationMode.WEIGHTED, @@ -184,8 +266,22 @@ def test_weighted_pass(self): def test_weighted_warn(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.6, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="ase", passed=True, score=0.5, mode=GateMode.BLOCK), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.6, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="ase", + passed=True, + score=0.5, + mode=GateMode.BLOCK, + ), ] policy = GatePolicy(combination=CombinationMode.WEIGHTED) rec, reason = apply_combination_logic(gates, policy) @@ -204,8 +300,22 @@ class TestScorecard: def test_create_scorecard(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.85, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.SECURITY, gate_name="security", policy_key="cisco", passed=True, score=1.0, mode=GateMode.WARN), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.85, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.SECURITY, + gate_name="security", + policy_key="cisco", + passed=True, + score=1.0, + mode=GateMode.WARN, + ), ] policy = GatePolicy() scorecard = Scorecard( @@ -225,9 +335,30 @@ def test_create_scorecard(self): def test_computed_fields(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.85, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.SECURITY, gate_name="security", policy_key="cisco", passed=False, score=0.3, mode=GateMode.BLOCK), - GateResult(gate_type=GateType.QUALITY, gate_name="quality", policy_key="llm-review", passed=False, score=0.4, mode=GateMode.WARN), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.85, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.SECURITY, + gate_name="security", + policy_key="cisco", + passed=False, + score=0.3, + mode=GateMode.BLOCK, + ), + GateResult( + gate_type=GateType.QUALITY, + gate_name="quality", + policy_key="llm-review", + passed=False, + score=0.4, + mode=GateMode.WARN, + ), ] scorecard = Scorecard( submission_name="test", @@ -245,7 +376,14 @@ def test_computed_fields(self): def test_scorecard_json_serialization(self): gates = [ - GateResult(gate_type=GateType.ENGINE, gate_name="evaluation", policy_key="harbor", passed=True, score=0.85, mode=GateMode.BLOCK), + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.85, + mode=GateMode.BLOCK, + ), ] scorecard = Scorecard( submission_name="test", @@ -255,7 +393,7 @@ def test_scorecard_json_serialization(self): policy=GatePolicy(), recommendation=Recommendation.PASS, recommendation_reason="OK", - created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + created_at=datetime(2025, 1, 1, 12, 0, 0, tzinfo=UTC), ) json_str = scorecard.model_dump_json() data = json.loads(json_str) @@ -322,7 +460,7 @@ def test_both_mode_reads_correct_reports(self, tmp_path): # Find each engine's gate by policy_key (implementation name) harbor_gate = next(g for g in engine_gates if g.policy_key == "harbor") ase_gate = next(g for g in engine_gates if g.policy_key == "ase") - + # Both should have category-based gate_name assert harbor_gate.gate_name == "evaluation" assert ase_gate.gate_name == "evaluation" @@ -373,8 +511,7 @@ def test_missing_validation_json_fails_validation(self, tmp_path): from abevalflow.certification import CheckId valid_structure_check = next( - c for c in scorecard.certification.foundational.checks - if c.check_id == CheckId.VALID_SKILL_STRUCTURE + c for c in scorecard.certification.foundational.checks if c.check_id == CheckId.VALID_SKILL_STRUCTURE ) assert valid_structure_check.passed is False @@ -414,8 +551,7 @@ def test_present_validation_json_valid_passes(self, tmp_path): from abevalflow.certification import CheckId valid_structure_check = next( - c for c in scorecard.certification.foundational.checks - if c.check_id == CheckId.VALID_SKILL_STRUCTURE + c for c in scorecard.certification.foundational.checks if c.check_id == CheckId.VALID_SKILL_STRUCTURE ) assert valid_structure_check.passed is True @@ -455,7 +591,6 @@ def test_present_validation_json_invalid_fails(self, tmp_path): from abevalflow.certification import CheckId valid_structure_check = next( - c for c in scorecard.certification.foundational.checks - if c.check_id == CheckId.VALID_SKILL_STRUCTURE + c for c in scorecard.certification.foundational.checks if c.check_id == CheckId.VALID_SKILL_STRUCTURE ) assert valid_structure_check.passed is False diff --git a/tests/test_skill_loader.py b/tests/test_skill_loader.py index db0480c..d64d431 100644 --- a/tests/test_skill_loader.py +++ b/tests/test_skill_loader.py @@ -98,6 +98,7 @@ def test_fetch_failure_returns_none( tmp_path: Path, ) -> None: import subprocess + mock_run.side_effect = subprocess.CalledProcessError(1, "git") target = tmp_path / "target" @@ -112,6 +113,7 @@ def test_fetch_timeout_returns_none( tmp_path: Path, ) -> None: import subprocess + mock_run.side_effect = subprocess.TimeoutExpired("git", 60) target = tmp_path / "target" diff --git a/tests/test_store_results.py b/tests/test_store_results.py index 8f4c59a..c38478f 100644 --- a/tests/test_store_results.py +++ b/tests/test_store_results.py @@ -2,17 +2,13 @@ from __future__ import annotations -import json -import uuid from pathlib import Path -from unittest.mock import MagicMock import pytest from sqlalchemy import create_engine, select from sqlalchemy.orm import sessionmaker from abevalflow.db.models import Base, EvaluationRun, Trial -from abevalflow.db.observer import ResultsObserver from abevalflow.report import ( AnalysisResult, AnalysisSummary, @@ -67,14 +63,8 @@ def _sample_result(name: str = "my-submission") -> AnalysisResult: recommendation=Recommendation.PASS, ), trials={ - "treatment": [ - TrialResult(trial_name=f"t-{i:03d}", reward=0.7 + 0.02 * i) - for i in range(5) - ], - "control": [ - TrialResult(trial_name=f"c-{i:03d}", reward=0.3 + 0.03 * i) - for i in range(5) - ], + "treatment": [TrialResult(trial_name=f"t-{i:03d}", reward=0.7 + 0.02 * i) for i in range(5)], + "control": [TrialResult(trial_name=f"c-{i:03d}", reward=0.3 + 0.03 * i) for i in range(5)], }, ) @@ -130,9 +120,7 @@ def test_map_trials(self): def test_map_trial_with_null_reward(self): result = _sample_result() - result.trials["treatment"].append( - TrialResult(trial_name="error-trial", reward=None) - ) + result.trials["treatment"].append(TrialResult(trial_name="error-trial", reward=None)) run = map_result_to_run(result, "run-001") trials = map_trials(result, run) error_trial = [t for t in trials if t.trial_name == "error-trial"][0] diff --git a/tests/test_validate.py b/tests/test_validate.py index dff9d63..e6828b9 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -8,11 +8,6 @@ import yaml from pydantic import ValidationError -from scripts.validate import ( - MAX_SUPPORTIVE_SIZE_BYTES, - main, - validate_submission, -) from abevalflow.schemas import ( CopySpec, ExperimentConfig, @@ -21,6 +16,11 @@ SubmissionMetadata, VariantSpec, ) +from scripts.validate import ( + MAX_SUPPORTIVE_SIZE_BYTES, + main, + validate_submission, +) VALID_METADATA = { "schema_version": "1.0", @@ -207,9 +207,7 @@ def test_valid_copy_list(self) -> None: assert len(vs.copy_dirs) == 2 def test_copy_dirs_via_field_name(self) -> None: - vs = VariantSpec( - copy_dirs=[CopySpec(src="skills", dest="/skills")] - ) + vs = VariantSpec(copy_dirs=[CopySpec(src="skills", dest="/skills")]) assert len(vs.copy_dirs) == 1 def test_duplicate_src_rejected(self) -> None: @@ -222,9 +220,7 @@ def test_duplicate_src_rejected(self) -> None: ) def test_env_from_secrets(self) -> None: - vs = VariantSpec( - env_from_secrets={"ANTHROPIC_API_KEY": "llm-keys/anthropic-key"} - ) + vs = VariantSpec(env_from_secrets={"ANTHROPIC_API_KEY": "llm-keys/anthropic-key"}) assert vs.env_from_secrets["ANTHROPIC_API_KEY"] == "llm-keys/anthropic-key" @@ -276,12 +272,8 @@ def test_custom_treatment_and_control(self) -> None: ec = ExperimentConfig( type="model", n_trials=10, - treatment=VariantSpec( - env_from_secrets={"MODEL_ENDPOINT": "models/endpoint-a"} - ), - control=VariantSpec( - env_from_secrets={"MODEL_ENDPOINT": "models/endpoint-b"} - ), + treatment=VariantSpec(env_from_secrets={"MODEL_ENDPOINT": "models/endpoint-a"}), + control=VariantSpec(env_from_secrets={"MODEL_ENDPOINT": "models/endpoint-b"}), ) assert ec.type == ExperimentType.MODEL assert ec.n_trials == 10 @@ -311,12 +303,8 @@ def test_explicit_experiment_config(self) -> None: "experiment": { "type": "model", "n_trials": 5, - "treatment": { - "env_from_secrets": {"MODEL": "models/gpt-4o"} - }, - "control": { - "env_from_secrets": {"MODEL": "models/gpt-3.5"} - }, + "treatment": {"env_from_secrets": {"MODEL": "models/gpt-4o"}}, + "control": {"env_from_secrets": {"MODEL": "models/gpt-3.5"}}, }, } model = SubmissionMetadata(**data) @@ -328,9 +316,7 @@ def test_experiment_with_copy_specs(self) -> None: **VALID_METADATA, "experiment": { "type": "custom", - "treatment": { - "copy": [{"src": "skills", "dest": "/skills"}] - }, + "treatment": {"copy": [{"src": "skills", "dest": "/skills"}]}, }, } model = SubmissionMetadata(**data)