From 28ccd36d8a1b214de7139960c35d6b18f5f4a634 Mon Sep 17 00:00:00 2001 From: apetruni Date: Mon, 6 Jul 2026 15:11:45 +0300 Subject: [PATCH 1/4] feat: add observability DB schema (Phase 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 4 new normalized tables for scorecard, gate results, certification, and observability metrics — enabling SQL queries over data that previously lived only as JSON files on MinIO. - ScorecardRow, GateResultRow, CertificationRow, ObservabilityMetricsRow models - 4 Alembic migrations (001-004) - Scorecard persistence in store_results.py with mapping functions - Backfill script for existing MinIO artifacts - Kubernetes migration job manifest - 24 new tests (693 total passing) APPENG-5370 Co-Authored-By: Claude Opus 4.6 (1M context) --- abevalflow/db/__init__.py | 20 +- abevalflow/db/models.py | 160 +++++++++- alembic.ini | 37 +++ alembic/env.py | 40 +++ alembic/script.py.mako | 25 ++ alembic/versions/001_add_scorecards_table.py | 52 ++++ .../versions/002_add_gate_results_table.py | 56 ++++ .../versions/003_add_certifications_table.py | 49 +++ .../004_add_observability_metrics_table.py | 51 +++ config/migrations/migration-job.yaml | 43 +++ pyproject.toml | 1 + scripts/backfill_scorecards.py | 236 ++++++++++++++ scripts/store_results.py | 98 +++++- tests/test_observability_models.py | 294 ++++++++++++++++++ tests/test_store_results_observability.py | 207 ++++++++++++ uv.lock | 38 ++- 16 files changed, 1394 insertions(+), 13 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/001_add_scorecards_table.py create mode 100644 alembic/versions/002_add_gate_results_table.py create mode 100644 alembic/versions/003_add_certifications_table.py create mode 100644 alembic/versions/004_add_observability_metrics_table.py create mode 100644 config/migrations/migration-job.yaml create mode 100644 scripts/backfill_scorecards.py create mode 100644 tests/test_observability_models.py create mode 100644 tests/test_store_results_observability.py diff --git a/abevalflow/db/__init__.py b/abevalflow/db/__init__.py index 674ad5e..66039b8 100644 --- a/abevalflow/db/__init__.py +++ b/abevalflow/db/__init__.py @@ -1,11 +1,29 @@ """Database models and engine for A/B evaluation results persistence.""" from abevalflow.db.engine import Session, get_engine, init_db -from abevalflow.db.models import Base, EvaluationRun, Trial +from abevalflow.db.models import ( + Base, + CertificationRow, + EvaluationRun, + GateResultRow, + MCPCheckerRun, + MCPCheckerTask, + ObservabilityMetricsRow, + ScorecardRow, + SecurityScan, + Trial, +) __all__ = [ "Base", + "CertificationRow", "EvaluationRun", + "GateResultRow", + "MCPCheckerRun", + "MCPCheckerTask", + "ObservabilityMetricsRow", + "ScorecardRow", + "SecurityScan", "Session", "Trial", "get_engine", diff --git a/abevalflow/db/models.py b/abevalflow/db/models.py index 85240a1..6a96a8b 100644 --- a/abevalflow/db/models.py +++ b/abevalflow/db/models.py @@ -1,9 +1,14 @@ """SQLAlchemy models for evaluation results persistence. -Three tables: +Tables: - ``evaluation_runs``: one row per pipeline run (flattened summary for fast queries) - ``trials``: one row per trial (drill-down into individual outcomes) - ``security_scans``: one row per security scan (independent of evaluation runs) +- ``mcpchecker_results`` / ``mcpchecker_tasks``: MCP server evaluation results +- ``scorecards``: one row per pipeline run (unified gate verdict) +- ``gate_results``: one row per gate per run (normalized gate details) +- ``certifications``: one row per certification level per run (max 3) +- ``observability_metrics``: token/timing metrics per run """ from __future__ import annotations @@ -12,14 +17,17 @@ from datetime import UTC, datetime from sqlalchemy import ( + BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, + Numeric, String, Text, + UniqueConstraint, Uuid, ) from sqlalchemy.dialects import postgresql @@ -253,3 +261,153 @@ def __repr__(self) -> str: f"" ) + + +class ScorecardRow(Base): + """One row per pipeline run with unified gate verdict.""" + + __tablename__ = "scorecards" + + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) + pipeline_run_id: Mapped[str] = mapped_column(String(255), unique=True, nullable=False) + submission_name: Mapped[str] = mapped_column(String(255), nullable=False) + eval_engine: Mapped[str] = mapped_column(String(50), nullable=False) + + recommendation: Mapped[str] = mapped_column(String(10), nullable=False) + recommendation_reason: Mapped[str] = mapped_column(Text, nullable=False) + combination_mode: Mapped[str] = mapped_column(String(20), nullable=False) + + gates_passed: Mapped[int] = mapped_column(Integer, nullable=False) + gates_failed: Mapped[int] = mapped_column(Integer, nullable=False) + blocking_gates_passed: Mapped[int] = mapped_column(Integer, nullable=False) + blocking_gates_failed: Mapped[int] = mapped_column(Integer, nullable=False) + + highest_certification: Mapped[str] = mapped_column(String(20), nullable=False) + scorecard_json: Mapped[dict] = mapped_column(_JsonVariant, nullable=False) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) + + gate_results: Mapped[list[GateResultRow]] = relationship( + back_populates="scorecard", cascade="all, delete-orphan" + ) + certifications: Mapped[list[CertificationRow]] = relationship( + back_populates="scorecard", cascade="all, delete-orphan" + ) + + __table_args__ = ( + Index("ix_scorecards_submission_name", "submission_name"), + Index("ix_scorecards_submission_created", "submission_name", "created_at"), + Index("ix_scorecards_highest_certification", "highest_certification"), + ) + + def __repr__(self) -> str: + return f"" + + +class GateResultRow(Base): + """One row per gate per run for normalized gate queries.""" + + __tablename__ = "gate_results" + + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) + scorecard_id: Mapped[uuid.UUID] = mapped_column( + Uuid, ForeignKey("scorecards.id", ondelete="CASCADE"), nullable=False + ) + + gate_name: Mapped[str] = mapped_column(String(100), nullable=False) + gate_type: Mapped[str] = mapped_column(String(20), nullable=False) + policy_key: Mapped[str | None] = mapped_column(String(100)) + passed: Mapped[bool] = mapped_column(Boolean, nullable=False) + score: Mapped[float] = mapped_column(Float, nullable=False) + mode: Mapped[str] = mapped_column(String(20), nullable=False) + threshold: Mapped[float | None] = mapped_column(Float) + findings_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + message: Mapped[str | None] = mapped_column(Text) + details_json: Mapped[dict | None] = mapped_column(_JsonVariant) + duration_ms: Mapped[int | None] = mapped_column(Integer) + prompt_tokens: Mapped[int | None] = mapped_column(Integer) + completion_tokens: Mapped[int | None] = mapped_column(Integer) + evaluated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + scorecard: Mapped[ScorecardRow] = relationship(back_populates="gate_results") + + __table_args__ = ( + Index("ix_gate_results_scorecard_id", "scorecard_id"), + Index("ix_gate_results_scorecard_policy", "scorecard_id", "policy_key"), + Index("ix_gate_results_gate_passed", "gate_name", "passed"), + Index("ix_gate_results_policy_key", "policy_key"), + ) + + def __repr__(self) -> str: + return f"" + + +class CertificationRow(Base): + """One row per certification level per run (max 3 per scorecard).""" + + __tablename__ = "certifications" + + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) + scorecard_id: Mapped[uuid.UUID] = mapped_column( + Uuid, ForeignKey("scorecards.id", ondelete="CASCADE"), nullable=False + ) + + level: Mapped[str] = mapped_column(String(20), nullable=False) + passed: Mapped[bool] = mapped_column(Boolean, nullable=False) + checks_total: Mapped[int] = mapped_column(Integer, nullable=False) + checks_passed: Mapped[int] = mapped_column(Integer, nullable=False) + checks_failed: Mapped[int] = mapped_column(Integer, nullable=False) + failed_checks: Mapped[list | None] = mapped_column(_JsonVariant) + details_json: Mapped[dict | None] = mapped_column(_JsonVariant) + + scorecard: Mapped[ScorecardRow] = relationship(back_populates="certifications") + + __table_args__ = ( + Index("ix_certifications_scorecard_id", "scorecard_id"), + Index("ix_certifications_level_passed", "level", "passed"), + ) + + def __repr__(self) -> str: + return f"" + + +class ObservabilityMetricsRow(Base): + """Token/timing metrics per pipeline run.""" + + __tablename__ = "observability_metrics" + + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) + pipeline_run_id: Mapped[str] = mapped_column(String(255), nullable=False) + submission_name: Mapped[str] = mapped_column(String(255), nullable=False) + + model_name: Mapped[str | None] = mapped_column(String(100)) + pipeline_duration_ms: Mapped[int | None] = mapped_column(Integer) + prepare_duration_ms: Mapped[int | None] = mapped_column(Integer) + test_duration_ms: Mapped[int | None] = mapped_column(Integer) + evaluate_duration_ms: Mapped[int | None] = mapped_column(Integer) + analyze_duration_ms: Mapped[int | None] = mapped_column(Integer) + store_duration_ms: Mapped[int | None] = mapped_column(Integer) + + total_prompt_tokens: Mapped[int | None] = mapped_column(BigInteger) + total_completion_tokens: Mapped[int | None] = mapped_column(BigInteger) + total_tokens: Mapped[int | None] = mapped_column(BigInteger) + estimated_cost_usd: Mapped[float | None] = mapped_column(Numeric(12, 6)) + + llm_calls_count: Mapped[int | None] = mapped_column(Integer) + trials_count: Mapped[int | None] = mapped_column(Integer) + attempt_number: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) + + __table_args__ = ( + UniqueConstraint("pipeline_run_id", "attempt_number", name="uq_obs_metrics_run_attempt"), + Index("ix_observability_metrics_submission_name", "submission_name"), + Index("ix_observability_metrics_created_at", "created_at"), + Index("ix_observability_metrics_model_name", "model_name"), + ) + + def __repr__(self) -> str: + return ( + f"" + ) diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..9a4af03 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,37 @@ +[alembic] +script_location = alembic +file_template = %%(rev)s_%%(slug)s +sqlalchemy.url = postgresql+psycopg://localhost/abevalflow + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..1a97793 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,40 @@ +"""Alembic environment configuration.""" + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine + +from abevalflow.db.models import Base + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + +db_url = os.environ.get("DATABASE_URL") +if db_url: + config.set_main_option("sqlalchemy.url", db_url) + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=target_metadata, literal_binds=True) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = create_engine(config.get_main_option("sqlalchemy.url")) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..958df87 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_add_scorecards_table.py b/alembic/versions/001_add_scorecards_table.py new file mode 100644 index 0000000..88a2236 --- /dev/null +++ b/alembic/versions/001_add_scorecards_table.py @@ -0,0 +1,52 @@ +"""Add scorecards table. + +Revision ID: 001 +Revises: None +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scorecards", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("pipeline_run_id", sa.String(255), nullable=False), + sa.Column("submission_name", sa.String(255), nullable=False), + sa.Column("eval_engine", sa.String(50), nullable=False), + sa.Column("recommendation", sa.String(10), nullable=False), + sa.Column("recommendation_reason", sa.Text(), nullable=False), + sa.Column("combination_mode", sa.String(20), nullable=False), + sa.Column("gates_passed", sa.Integer(), nullable=False), + sa.Column("gates_failed", sa.Integer(), nullable=False), + sa.Column("blocking_gates_passed", sa.Integer(), nullable=False), + sa.Column("blocking_gates_failed", sa.Integer(), nullable=False), + sa.Column("highest_certification", sa.String(20), nullable=False), + sa.Column( + "scorecard_json", + sa.JSON().with_variant(postgresql.JSONB(), "postgresql"), + nullable=False, + ), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("pipeline_run_id"), + ) + op.create_index("ix_scorecards_submission_name", "scorecards", ["submission_name"]) + op.create_index("ix_scorecards_submission_created", "scorecards", ["submission_name", "created_at"]) + op.create_index("ix_scorecards_highest_certification", "scorecards", ["highest_certification"]) + + +def downgrade() -> None: + op.drop_index("ix_scorecards_highest_certification", table_name="scorecards") + op.drop_index("ix_scorecards_submission_created", table_name="scorecards") + op.drop_index("ix_scorecards_submission_name", table_name="scorecards") + op.drop_table("scorecards") diff --git a/alembic/versions/002_add_gate_results_table.py b/alembic/versions/002_add_gate_results_table.py new file mode 100644 index 0000000..6468c57 --- /dev/null +++ b/alembic/versions/002_add_gate_results_table.py @@ -0,0 +1,56 @@ +"""Add gate_results table. + +Revision ID: 002 +Revises: 001 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "002" +down_revision: Union[str, None] = "001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "gate_results", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("scorecard_id", sa.Uuid(), nullable=False), + sa.Column("gate_name", sa.String(100), nullable=False), + sa.Column("gate_type", sa.String(20), nullable=False), + sa.Column("policy_key", sa.String(100), nullable=True), + sa.Column("passed", sa.Boolean(), nullable=False), + sa.Column("score", sa.Float(), nullable=False), + sa.Column("mode", sa.String(20), nullable=False), + sa.Column("threshold", sa.Float(), nullable=True), + sa.Column("findings_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("message", sa.Text(), nullable=True), + sa.Column( + "details_json", + sa.JSON().with_variant(postgresql.JSONB(), "postgresql"), + nullable=True, + ), + sa.Column("duration_ms", sa.Integer(), nullable=True), + sa.Column("prompt_tokens", sa.Integer(), nullable=True), + sa.Column("completion_tokens", sa.Integer(), nullable=True), + sa.Column("evaluated_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["scorecard_id"], ["scorecards.id"], ondelete="CASCADE"), + ) + op.create_index("ix_gate_results_scorecard_id", "gate_results", ["scorecard_id"]) + op.create_index("ix_gate_results_scorecard_policy", "gate_results", ["scorecard_id", "policy_key"]) + op.create_index("ix_gate_results_gate_passed", "gate_results", ["gate_name", "passed"]) + op.create_index("ix_gate_results_policy_key", "gate_results", ["policy_key"]) + + +def downgrade() -> None: + op.drop_index("ix_gate_results_policy_key", table_name="gate_results") + op.drop_index("ix_gate_results_gate_passed", table_name="gate_results") + op.drop_index("ix_gate_results_scorecard_policy", table_name="gate_results") + op.drop_index("ix_gate_results_scorecard_id", table_name="gate_results") + op.drop_table("gate_results") diff --git a/alembic/versions/003_add_certifications_table.py b/alembic/versions/003_add_certifications_table.py new file mode 100644 index 0000000..a51dbb8 --- /dev/null +++ b/alembic/versions/003_add_certifications_table.py @@ -0,0 +1,49 @@ +"""Add certifications table. + +Revision ID: 003 +Revises: 002 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "003" +down_revision: Union[str, None] = "002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "certifications", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("scorecard_id", sa.Uuid(), nullable=False), + sa.Column("level", sa.String(20), nullable=False), + sa.Column("passed", sa.Boolean(), nullable=False), + sa.Column("checks_total", sa.Integer(), nullable=False), + sa.Column("checks_passed", sa.Integer(), nullable=False), + sa.Column("checks_failed", sa.Integer(), nullable=False), + sa.Column( + "failed_checks", + sa.JSON().with_variant(postgresql.JSONB(), "postgresql"), + nullable=True, + ), + sa.Column( + "details_json", + sa.JSON().with_variant(postgresql.JSONB(), "postgresql"), + nullable=True, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint(["scorecard_id"], ["scorecards.id"], ondelete="CASCADE"), + ) + op.create_index("ix_certifications_scorecard_id", "certifications", ["scorecard_id"]) + op.create_index("ix_certifications_level_passed", "certifications", ["level", "passed"]) + + +def downgrade() -> None: + op.drop_index("ix_certifications_level_passed", table_name="certifications") + op.drop_index("ix_certifications_scorecard_id", table_name="certifications") + op.drop_table("certifications") diff --git a/alembic/versions/004_add_observability_metrics_table.py b/alembic/versions/004_add_observability_metrics_table.py new file mode 100644 index 0000000..af4f597 --- /dev/null +++ b/alembic/versions/004_add_observability_metrics_table.py @@ -0,0 +1,51 @@ +"""Add observability_metrics table. + +Revision ID: 004 +Revises: 003 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "004" +down_revision: Union[str, None] = "003" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "observability_metrics", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("pipeline_run_id", sa.String(255), nullable=False), + sa.Column("submission_name", sa.String(255), nullable=False), + sa.Column("model_name", sa.String(100), nullable=True), + sa.Column("pipeline_duration_ms", sa.Integer(), nullable=True), + sa.Column("prepare_duration_ms", sa.Integer(), nullable=True), + sa.Column("test_duration_ms", sa.Integer(), nullable=True), + sa.Column("evaluate_duration_ms", sa.Integer(), nullable=True), + sa.Column("analyze_duration_ms", sa.Integer(), nullable=True), + sa.Column("store_duration_ms", sa.Integer(), nullable=True), + sa.Column("total_prompt_tokens", sa.BigInteger(), nullable=True), + sa.Column("total_completion_tokens", sa.BigInteger(), nullable=True), + sa.Column("total_tokens", sa.BigInteger(), nullable=True), + sa.Column("estimated_cost_usd", sa.Numeric(12, 6), nullable=True), + sa.Column("llm_calls_count", sa.Integer(), nullable=True), + sa.Column("trials_count", sa.Integer(), nullable=True), + sa.Column("attempt_number", sa.Integer(), nullable=False, server_default="1"), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("pipeline_run_id", "attempt_number", name="uq_obs_metrics_run_attempt"), + ) + op.create_index("ix_observability_metrics_submission_name", "observability_metrics", ["submission_name"]) + op.create_index("ix_observability_metrics_created_at", "observability_metrics", ["created_at"]) + op.create_index("ix_observability_metrics_model_name", "observability_metrics", ["model_name"]) + + +def downgrade() -> None: + op.drop_index("ix_observability_metrics_model_name", table_name="observability_metrics") + op.drop_index("ix_observability_metrics_created_at", table_name="observability_metrics") + op.drop_index("ix_observability_metrics_submission_name", table_name="observability_metrics") + op.drop_table("observability_metrics") diff --git a/config/migrations/migration-job.yaml b/config/migrations/migration-job.yaml new file mode 100644 index 0000000..6c46902 --- /dev/null +++ b/config/migrations/migration-job.yaml @@ -0,0 +1,43 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: abevalflow-migrate + namespace: ab-eval-flow + labels: + app: abevalflow + component: migration +spec: + backoffLimit: 3 + template: + metadata: + labels: + app: abevalflow + component: migration + spec: + restartPolicy: OnFailure + containers: + - name: migrate + image: registry.access.redhat.com/ubi9/python-311:latest + command: + - /bin/bash + - -c + - | + set -e + pip install --quiet alembic sqlalchemy "psycopg[binary]" pydantic tenacity + git clone https://github.com/RHEcosystemAppEng/ABEvalFlow.git /tmp/abevalflow + cd /tmp/abevalflow + alembic upgrade head + echo "Migration complete" + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: ab-eval-db-credentials + key: database-url + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/pyproject.toml b/pyproject.toml index 5ea1245..13e76ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "kubernetes>=28.0", "tenacity>=8.2", "sqlalchemy>=2.0", + "alembic>=1.13", "psycopg[binary]>=3.1", "minio>=7.0", "openai>=1.0", diff --git a/scripts/backfill_scorecards.py b/scripts/backfill_scorecards.py new file mode 100644 index 0000000..745b0c8 --- /dev/null +++ b/scripts/backfill_scorecards.py @@ -0,0 +1,236 @@ +"""Backfill scorecards, gate_results, and certifications from MinIO artifacts. + +Reads scorecard.json files from MinIO and inserts into the new normalized +tables. Idempotent — skips scorecards that already exist (by pipeline_run_id). + +Usage:: + + python scripts/backfill_scorecards.py \\ + --database-url postgresql+psycopg://user:pass@host:5432/abevalflow \\ + --minio-endpoint minio.ab-eval-flow.svc:9000 \\ + --minio-access-key abevalflow \\ + --minio-secret-key \\ + --bucket ab-eval-reports + + # Dry run (validate only, no writes): + python scripts/backfill_scorecards.py --dry-run ... + + # Resume from checkpoint: + python scripts/backfill_scorecards.py --resume ... + + # Limit to N artifacts: + python scripts/backfill_scorecards.py --limit 100 ... + + # Filter by prefix: + python scripts/backfill_scorecards.py --prefix certified/ ... +""" + +from __future__ import annotations + +import argparse +import io +import json +import logging +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path + +from sqlalchemy import select + +from abevalflow.db.engine import get_engine, init_db, make_session +from abevalflow.db.models import CertificationRow, GateResultRow, ScorecardRow +from abevalflow.scorecard import Scorecard +from scripts.store_results import map_certifications, map_gate_results, map_scorecard_to_row + +logger = logging.getLogger(__name__) + + +@dataclass +class BackfillState: + last_processed_key: str = "" + processed_count: int = 0 + skipped_count: int = 0 + error_count: int = 0 + errors: list[dict] = field(default_factory=list) + + +def _load_checkpoint(path: Path) -> BackfillState: + if path.exists(): + data = json.loads(path.read_text()) + return BackfillState( + last_processed_key=data.get("last_processed_key", ""), + processed_count=data.get("processed_count", 0), + skipped_count=data.get("skipped_count", 0), + error_count=data.get("error_count", 0), + errors=data.get("errors", []), + ) + return BackfillState() + + +def _save_checkpoint(path: Path, state: BackfillState) -> None: + path.write_text(json.dumps({ + "last_processed_key": state.last_processed_key, + "processed_count": state.processed_count, + "skipped_count": state.skipped_count, + "error_count": state.error_count, + "errors": state.errors[-50:], + }, indent=2)) + + +def backfill( + database_url: str | None = None, + minio_endpoint: str | None = None, + minio_access_key: str | None = None, + minio_secret_key: str | None = None, + bucket: str = "ab-eval-reports", + secure: bool = False, + dry_run: bool = False, + resume: bool = False, + limit: int | None = None, + prefix_filter: str | None = None, + checkpoint_path: Path | None = None, +) -> BackfillState: + from minio import Minio + + endpoint = minio_endpoint or os.environ.get("MINIO_ENDPOINT", "localhost:9000") + access_key = minio_access_key or os.environ.get("MINIO_ACCESS_KEY", "") + secret_key = minio_secret_key or os.environ.get("MINIO_SECRET_KEY", "") + + client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure) + + if not client.bucket_exists(bucket): + logger.error("Bucket %s does not exist", bucket) + return BackfillState(error_count=1) + + ckpt_path = checkpoint_path or Path("_backfill_checkpoint.json") + state = _load_checkpoint(ckpt_path) if resume else BackfillState() + + engine = get_engine(database_url) + init_db(engine) + session_factory = make_session(engine) + + objects = client.list_objects(bucket, prefix=prefix_filter, recursive=True) + scorecard_keys = sorted( + obj.object_name + for obj in objects + if obj.object_name and obj.object_name.endswith("/scorecard.json") + ) + + if resume and state.last_processed_key: + try: + start_idx = scorecard_keys.index(state.last_processed_key) + 1 + scorecard_keys = scorecard_keys[start_idx:] + logger.info("Resuming from %s (%d remaining)", state.last_processed_key, len(scorecard_keys)) + except ValueError: + logger.warning("Checkpoint key not found in bucket, starting from beginning") + + if limit: + scorecard_keys = scorecard_keys[:limit] + + logger.info("Found %d scorecard.json artifacts to process", len(scorecard_keys)) + + for key in scorecard_keys: + try: + response = client.get_object(bucket, key) + raw = response.read() + response.close() + response.release_conn() + + scorecard = Scorecard.model_validate_json(raw) + + with session_factory() as session: + existing = session.execute( + select(ScorecardRow).where( + ScorecardRow.pipeline_run_id == scorecard.pipeline_run_id + ) + ).scalar_one_or_none() + + if existing is not None: + logger.debug("Skipping %s — already exists", scorecard.pipeline_run_id) + state.skipped_count += 1 + state.last_processed_key = key + continue + + if dry_run: + logger.info("[DRY RUN] Would insert scorecard for %s", scorecard.pipeline_run_id) + state.processed_count += 1 + state.last_processed_key = key + continue + + sc_row = map_scorecard_to_row(scorecard) + gate_rows = map_gate_results(scorecard, sc_row) + cert_rows = map_certifications(scorecard, sc_row) + + session.add(sc_row) + session.add_all(gate_rows) + session.add_all(cert_rows) + session.commit() + + state.processed_count += 1 + logger.info( + "Backfilled: %s (gates=%d, certs=%d)", + scorecard.pipeline_run_id, + len(gate_rows), + len(cert_rows), + ) + + except Exception as e: + state.error_count += 1 + state.errors.append({"key": key, "error": str(e)[:200]}) + logger.warning("Error processing %s: %s", key, str(e)[:200]) + + state.last_processed_key = key + + if state.processed_count % 50 == 0 and state.processed_count > 0: + _save_checkpoint(ckpt_path, state) + + _save_checkpoint(ckpt_path, state) + + logger.info( + "Backfill complete: processed=%d skipped=%d errors=%d", + state.processed_count, + state.skipped_count, + state.error_count, + ) + return state + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + parser = argparse.ArgumentParser(description="Backfill scorecards from MinIO artifacts") + parser.add_argument("--database-url", type=str, default=None) + parser.add_argument("--minio-endpoint", type=str, default=None) + parser.add_argument("--minio-access-key", type=str, default=None) + parser.add_argument("--minio-secret-key", type=str, default=None) + parser.add_argument("--bucket", type=str, default="ab-eval-reports") + parser.add_argument("--secure", action="store_true", default=False) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--resume", action="store_true") + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--prefix", type=str, default=None) + parser.add_argument("--checkpoint", type=Path, default=None) + args = parser.parse_args() + + state = backfill( + database_url=args.database_url, + minio_endpoint=args.minio_endpoint, + minio_access_key=args.minio_access_key, + minio_secret_key=args.minio_secret_key, + bucket=args.bucket, + secure=args.secure, + dry_run=args.dry_run, + resume=args.resume, + limit=args.limit, + prefix_filter=args.prefix, + checkpoint_path=args.checkpoint, + ) + + sys.exit(0 if state.error_count == 0 else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/store_results.py b/scripts/store_results.py index ee9af3a..80b0d60 100644 --- a/scripts/store_results.py +++ b/scripts/store_results.py @@ -26,10 +26,20 @@ from sqlalchemy.exc import IntegrityError from abevalflow.db.engine import get_engine, init_db, make_session -from abevalflow.db.models import EvaluationRun, MCPCheckerRun, MCPCheckerTask, SecurityScan, Trial +from abevalflow.db.models import ( + CertificationRow, + EvaluationRun, + GateResultRow, + MCPCheckerRun, + MCPCheckerTask, + ScorecardRow, + SecurityScan, + Trial, +) from abevalflow.db.observer import discover_observers, notify_observers from abevalflow.mcpchecker_report import MCPCheckerResult from abevalflow.report import AnalysisResult +from abevalflow.scorecard import Scorecard logger = logging.getLogger(__name__) @@ -121,6 +131,74 @@ def map_security_scans( return scans +def map_scorecard_to_row(scorecard: Scorecard) -> ScorecardRow: + """Flatten a Scorecard Pydantic model into a ScorecardRow.""" + return ScorecardRow( + pipeline_run_id=scorecard.pipeline_run_id, + submission_name=scorecard.submission_name, + eval_engine=scorecard.eval_engine, + recommendation=scorecard.recommendation.value, + recommendation_reason=scorecard.recommendation_reason, + combination_mode=scorecard.policy.combination.value, + gates_passed=scorecard.gates_passed, + gates_failed=scorecard.gates_failed, + blocking_gates_passed=scorecard.blocking_gates_passed, + blocking_gates_failed=scorecard.blocking_gates_failed, + highest_certification=scorecard.highest_certification.value, + scorecard_json=json.loads(scorecard.model_dump_json()), + ) + + +def map_gate_results(scorecard: Scorecard, scorecard_row: ScorecardRow) -> list[GateResultRow]: + """Create GateResultRow rows from scorecard gates.""" + rows: list[GateResultRow] = [] + for gate in scorecard.gates: + rows.append( + GateResultRow( + scorecard=scorecard_row, + gate_name=gate.gate_name, + gate_type=gate.gate_type.value, + policy_key=gate.get_policy_key(), + passed=gate.passed, + score=gate.score, + mode=gate.mode.value, + threshold=gate.threshold, + findings_count=len(gate.findings), + message=gate.message, + details_json=gate.details if gate.details else None, + evaluated_at=gate.evaluated_at, + ) + ) + return rows + + +def map_certifications(scorecard: Scorecard, scorecard_row: ScorecardRow) -> list[CertificationRow]: + """Create CertificationRow rows from scorecard certification results.""" + if scorecard.certification is None: + return [] + + rows: list[CertificationRow] = [] + for level_result in [ + scorecard.certification.foundational, + scorecard.certification.trusted, + scorecard.certification.certified, + ]: + failed = [c.check_id.value for c in level_result.checks if not c.passed] + rows.append( + CertificationRow( + scorecard=scorecard_row, + level=level_result.level.value, + passed=level_result.passed, + checks_total=level_result.checks_total, + checks_passed=level_result.checks_passed, + checks_failed=level_result.checks_total - level_result.checks_passed, + failed_checks=failed if failed else None, + details_json=json.loads(level_result.model_dump_json()), + ) + ) + return rows + + def map_mcpchecker_result(result: MCPCheckerResult, run_id: str) -> MCPCheckerRun: """Create MCPCheckerRun row from MCPCheckerResult.""" return MCPCheckerRun( @@ -295,6 +373,24 @@ def store( "Security scans already exist for run %s, skipping", effective_run_id, ) + scorecard_path = report_dir / "scorecard.json" + if scorecard_path.exists(): + try: + scorecard = Scorecard.model_validate_json(scorecard_path.read_bytes()) + sc_row = map_scorecard_to_row(scorecard) + gate_rows = map_gate_results(scorecard, sc_row) + cert_rows = map_certifications(scorecard, sc_row) + session.add(sc_row) + session.add_all(gate_rows) + session.add_all(cert_rows) + logger.info( + "Scorecard queued: gates=%d certifications=%d", + len(gate_rows), + len(cert_rows), + ) + except Exception: + logger.warning("Failed to persist scorecard — continuing without", exc_info=True) + try: session.commit() except IntegrityError as e: diff --git a/tests/test_observability_models.py b/tests/test_observability_models.py new file mode 100644 index 0000000..64c61f7 --- /dev/null +++ b/tests/test_observability_models.py @@ -0,0 +1,294 @@ +"""Tests for observability DB models (scorecards, gate_results, certifications, observability_metrics).""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from decimal import Decimal + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session, sessionmaker + +from abevalflow.db.models import ( + Base, + CertificationRow, + GateResultRow, + ObservabilityMetricsRow, + ScorecardRow, +) + + +@pytest.fixture() +def engine(): + eng = create_engine("sqlite://", connect_args={"check_same_thread": False}) + Base.metadata.create_all(eng) + return eng + + +@pytest.fixture() +def session(engine) -> Session: + factory = sessionmaker(bind=engine) + with factory() as s: + yield s + + +def _make_scorecard(**overrides) -> ScorecardRow: + defaults = { + "pipeline_run_id": f"run-{uuid.uuid4().hex[:8]}", + "submission_name": "test-skill", + "eval_engine": "harbor", + "recommendation": "pass", + "recommendation_reason": "All gates passed", + "combination_mode": "all_pass", + "gates_passed": 3, + "gates_failed": 0, + "blocking_gates_passed": 2, + "blocking_gates_failed": 0, + "highest_certification": "trusted", + "scorecard_json": {"recommendation": "pass", "gates": []}, + } + defaults.update(overrides) + return ScorecardRow(**defaults) + + +def _make_gate_result(scorecard: ScorecardRow, **overrides) -> GateResultRow: + defaults = { + "scorecard": scorecard, + "gate_name": "evaluation", + "gate_type": "engine", + "policy_key": "harbor", + "passed": True, + "score": 0.85, + "mode": "block", + "threshold": 0.0, + "findings_count": 0, + } + defaults.update(overrides) + return GateResultRow(**defaults) + + +def _make_certification(scorecard: ScorecardRow, **overrides) -> CertificationRow: + defaults = { + "scorecard": scorecard, + "level": "foundational", + "passed": True, + "checks_total": 5, + "checks_passed": 5, + "checks_failed": 0, + } + defaults.update(overrides) + return CertificationRow(**defaults) + + +def _make_metrics(**overrides) -> ObservabilityMetricsRow: + defaults = { + "pipeline_run_id": f"run-{uuid.uuid4().hex[:8]}", + "submission_name": "test-skill", + "model_name": "claude-sonnet-4-6", + "total_prompt_tokens": 15000, + "total_completion_tokens": 3500, + "total_tokens": 18500, + "estimated_cost_usd": 0.097500, + "attempt_number": 1, + } + defaults.update(overrides) + return ObservabilityMetricsRow(**defaults) + + +class TestScorecardRow: + def test_create_and_read(self, session: Session) -> None: + sc = _make_scorecard() + session.add(sc) + session.commit() + + loaded = session.execute(select(ScorecardRow)).scalar_one() + assert loaded.submission_name == "test-skill" + assert loaded.recommendation == "pass" + assert loaded.highest_certification == "trusted" + + def test_created_at_default(self, session: Session) -> None: + sc = _make_scorecard() + session.add(sc) + session.commit() + + loaded = session.execute(select(ScorecardRow)).scalar_one() + assert loaded.created_at is not None + assert (datetime.now(UTC) - loaded.created_at.replace(tzinfo=UTC)).total_seconds() < 5 + + def test_unique_pipeline_run_id(self, session: Session) -> None: + run_id = "run-duplicate" + session.add(_make_scorecard(pipeline_run_id=run_id)) + session.commit() + + session.add(_make_scorecard(pipeline_run_id=run_id)) + with pytest.raises(Exception): + session.commit() + + def test_scorecard_json_round_trip(self, session: Session) -> None: + data = {"recommendation": "pass", "gates": [{"name": "security", "passed": True}]} + sc = _make_scorecard(scorecard_json=data) + session.add(sc) + session.commit() + + loaded = session.execute(select(ScorecardRow)).scalar_one() + assert loaded.scorecard_json == data + + def test_repr(self) -> None: + sc = _make_scorecard() + assert "test-skill" in repr(sc) + assert "pass" in repr(sc) + + +class TestGateResultRow: + def test_create_linked_to_scorecard(self, session: Session) -> None: + sc = _make_scorecard() + gate = _make_gate_result(sc) + session.add(sc) + session.add(gate) + session.commit() + + loaded = session.execute(select(GateResultRow)).scalar_one() + assert loaded.gate_name == "evaluation" + assert loaded.passed is True + assert loaded.score == 0.85 + + def test_nullable_fields(self, session: Session) -> None: + sc = _make_scorecard() + gate = _make_gate_result( + sc, + threshold=None, + message=None, + details_json=None, + duration_ms=None, + prompt_tokens=None, + completion_tokens=None, + evaluated_at=None, + ) + session.add(sc) + session.add(gate) + session.commit() + + loaded = session.execute(select(GateResultRow)).scalar_one() + assert loaded.threshold is None + assert loaded.duration_ms is None + assert loaded.prompt_tokens is None + + def test_cascade_delete(self, session: Session) -> None: + sc = _make_scorecard() + gate = _make_gate_result(sc) + session.add(sc) + session.add(gate) + session.commit() + + session.delete(sc) + session.commit() + + assert session.execute(select(GateResultRow)).scalar_one_or_none() is None + + def test_relationship_back_populates(self, session: Session) -> None: + sc = _make_scorecard() + g1 = _make_gate_result(sc, gate_name="evaluation") + g2 = _make_gate_result(sc, gate_name="security", policy_key="cisco", score=0.92) + session.add(sc) + session.add_all([g1, g2]) + session.commit() + + loaded = session.execute(select(ScorecardRow)).scalar_one() + assert len(loaded.gate_results) == 2 + + +class TestCertificationRow: + def test_create_all_three_levels(self, session: Session) -> None: + sc = _make_scorecard() + levels = [ + _make_certification(sc, level="foundational", passed=True), + _make_certification(sc, level="trusted", passed=True, checks_passed=4, checks_failed=1), + _make_certification(sc, level="certified", passed=False, checks_passed=2, checks_failed=3), + ] + session.add(sc) + session.add_all(levels) + session.commit() + + loaded = session.execute(select(CertificationRow)).scalars().all() + assert len(loaded) == 3 + + def test_cascade_delete(self, session: Session) -> None: + sc = _make_scorecard() + cert = _make_certification(sc) + session.add(sc) + session.add(cert) + session.commit() + + session.delete(sc) + session.commit() + + assert session.execute(select(CertificationRow)).scalar_one_or_none() is None + + def test_failed_checks_json(self, session: Session) -> None: + sc = _make_scorecard() + cert = _make_certification( + sc, + passed=False, + checks_passed=3, + checks_failed=2, + failed_checks=["basic_security_validation", "metadata_compliance"], + ) + session.add(sc) + session.add(cert) + session.commit() + + loaded = session.execute(select(CertificationRow)).scalar_one() + assert loaded.failed_checks == ["basic_security_validation", "metadata_compliance"] + + +class TestObservabilityMetricsRow: + def test_create_and_read(self, session: Session) -> None: + m = _make_metrics() + session.add(m) + session.commit() + + loaded = session.execute(select(ObservabilityMetricsRow)).scalar_one() + assert loaded.total_tokens == 18500 + assert loaded.model_name == "claude-sonnet-4-6" + + def test_unique_run_attempt(self, session: Session) -> None: + run_id = "run-unique-test" + session.add(_make_metrics(pipeline_run_id=run_id, attempt_number=1)) + session.commit() + + session.add(_make_metrics(pipeline_run_id=run_id, attempt_number=1)) + with pytest.raises(Exception): + session.commit() + + def test_multiple_attempts_allowed(self, session: Session) -> None: + run_id = "run-retry-test" + session.add(_make_metrics(pipeline_run_id=run_id, attempt_number=1)) + session.add(_make_metrics(pipeline_run_id=run_id, attempt_number=2)) + session.commit() + + loaded = session.execute(select(ObservabilityMetricsRow)).scalars().all() + assert len(loaded) == 2 + + def test_nullable_token_fields(self, session: Session) -> None: + m = _make_metrics( + total_prompt_tokens=None, + total_completion_tokens=None, + total_tokens=None, + estimated_cost_usd=None, + pipeline_duration_ms=None, + ) + session.add(m) + session.commit() + + loaded = session.execute(select(ObservabilityMetricsRow)).scalar_one() + assert loaded.total_tokens is None + assert loaded.estimated_cost_usd is None + + def test_numeric_cost_precision(self, session: Session) -> None: + m = _make_metrics(estimated_cost_usd=0.123456) + session.add(m) + session.commit() + + loaded = session.execute(select(ObservabilityMetricsRow)).scalar_one() + assert abs(float(loaded.estimated_cost_usd) - 0.123456) < 1e-6 diff --git a/tests/test_store_results_observability.py b/tests/test_store_results_observability.py new file mode 100644 index 0000000..4db9c99 --- /dev/null +++ b/tests/test_store_results_observability.py @@ -0,0 +1,207 @@ +"""Tests for scorecard persistence in store_results.py.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from abevalflow.db.models import ( + Base, + CertificationRow, + EvaluationRun, + GateResultRow, + ScorecardRow, +) +from abevalflow.gates.base import GateMode, GateResult, GateType +from abevalflow.report import ( + AnalysisResult, + AnalysisSummary, + Provenance, + Recommendation, + TrialResult, + VariantSummary, +) +from abevalflow.schemas import CombinationMode, GatePolicy +from abevalflow.scorecard import Recommendation as ScorecardRecommendation +from abevalflow.scorecard import Scorecard +from scripts.store_results import ( + map_certifications, + map_gate_results, + map_scorecard_to_row, + store, +) + + +def _sample_result(name: str = "my-submission") -> AnalysisResult: + return AnalysisResult( + submission_name=name, + provenance=Provenance( + commit_sha="abc123", + pipeline_run_id="tekton-run-001", + treatment_image_ref="registry/img@sha256:aaa", + control_image_ref="registry/img@sha256:bbb", + harbor_fork_revision="main", + ), + summary=AnalysisSummary( + treatment=VariantSummary( + n_trials=20, n_passed=16, n_failed=3, n_errors=1, + pass_rate=0.8, mean_reward=0.72, median_reward=0.75, std_reward=0.15, + ), + control=VariantSummary( + n_trials=20, n_passed=10, n_failed=8, n_errors=2, + pass_rate=0.5, mean_reward=0.45, median_reward=0.42, std_reward=0.20, + ), + uplift=0.3, mean_reward_gap=0.27, ttest_p_value=0.02, fisher_p_value=0.04, + recommendation=Recommendation.PASS, + ), + trials={ + "treatment": [TrialResult(trial_name=f"t-{i:03d}", reward=0.7 + 0.02 * i) for i in range(5)], + "control": [TrialResult(trial_name=f"c-{i:03d}", reward=0.3 + 0.03 * i) for i in range(5)], + }, + ) + + +def _sample_scorecard(run_id: str = "tekton-run-001") -> Scorecard: + return Scorecard( + submission_name="my-submission", + pipeline_run_id=run_id, + eval_engine="harbor", + gates=[ + GateResult( + gate_type=GateType.ENGINE, + gate_name="evaluation", + policy_key="harbor", + passed=True, + score=0.85, + mode=GateMode.BLOCK, + threshold=0.0, + message="Treatment outperforms control", + ), + GateResult( + gate_type=GateType.SECURITY, + gate_name="security", + policy_key="cisco", + passed=True, + score=1.0, + mode=GateMode.WARN, + ), + ], + policy=GatePolicy(combination=CombinationMode.ALL_PASS), + recommendation=ScorecardRecommendation.PASS, + recommendation_reason="All gates passed", + ) + + +def _write_report(tmp_path: Path, result: AnalysisResult) -> Path: + report_dir = tmp_path / "reports" + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / "report.json").write_text(result.model_dump_json(indent=2)) + return report_dir + + +def _write_scorecard(report_dir: Path, scorecard: Scorecard) -> None: + (report_dir / "scorecard.json").write_text(scorecard.model_dump_json(indent=2)) + + +@pytest.fixture() +def db_url(tmp_path): + return f"sqlite:///{tmp_path / 'test.db'}" + + +@pytest.fixture() +def session_factory(db_url): + engine = create_engine(db_url, connect_args={"check_same_thread": False}) + Base.metadata.create_all(engine) + return sessionmaker(bind=engine) + + +class TestScorecardMapping: + def test_map_scorecard_to_row(self) -> None: + sc = _sample_scorecard() + row = map_scorecard_to_row(sc) + assert row.submission_name == "my-submission" + assert row.pipeline_run_id == "tekton-run-001" + assert row.recommendation == "pass" + assert row.combination_mode == "all_pass" + assert row.gates_passed == 2 + assert row.gates_failed == 0 + assert row.highest_certification == "none" + assert row.scorecard_json["submission_name"] == "my-submission" + + def test_map_gate_results(self) -> None: + sc = _sample_scorecard() + sc_row = map_scorecard_to_row(sc) + gates = map_gate_results(sc, sc_row) + assert len(gates) == 2 + assert gates[0].gate_name == "evaluation" + assert gates[0].policy_key == "harbor" + assert gates[0].passed is True + assert gates[0].score == 0.85 + assert gates[1].gate_name == "security" + assert gates[1].policy_key == "cisco" + + def test_map_certifications_none(self) -> None: + sc = _sample_scorecard() + sc_row = map_scorecard_to_row(sc) + certs = map_certifications(sc, sc_row) + assert certs == [] + + +class TestStoreWithScorecard: + def test_store_with_scorecard_json(self, tmp_path: Path, db_url: str, session_factory) -> None: + result = _sample_result() + report_dir = _write_report(tmp_path, result) + scorecard = _sample_scorecard() + _write_scorecard(report_dir, scorecard) + + ok = store(report_dir, db_url, "tekton-run-001") + assert ok is True + + with session_factory() as session: + assert session.execute(select(EvaluationRun)).scalar_one() is not None + sc = session.execute(select(ScorecardRow)).scalar_one() + assert sc.submission_name == "my-submission" + assert sc.recommendation == "pass" + + gates = session.execute(select(GateResultRow)).scalars().all() + assert len(gates) == 2 + + def test_store_without_scorecard_json(self, tmp_path: Path, db_url: str, session_factory) -> None: + result = _sample_result() + report_dir = _write_report(tmp_path, result) + + ok = store(report_dir, db_url, "tekton-run-002") + assert ok is True + + with session_factory() as session: + assert session.execute(select(EvaluationRun)).scalar_one() is not None + assert session.execute(select(ScorecardRow)).scalar_one_or_none() is None + + def test_store_with_malformed_scorecard(self, tmp_path: Path, db_url: str, session_factory) -> None: + result = _sample_result() + report_dir = _write_report(tmp_path, result) + (report_dir / "scorecard.json").write_text("{ invalid json !!!") + + ok = store(report_dir, db_url, "tekton-run-003") + assert ok is True + + with session_factory() as session: + assert session.execute(select(EvaluationRun)).scalar_one() is not None + assert session.execute(select(ScorecardRow)).scalar_one_or_none() is None + + def test_store_idempotent_with_scorecard(self, tmp_path: Path, db_url: str, session_factory) -> None: + result = _sample_result() + report_dir = _write_report(tmp_path, result) + scorecard = _sample_scorecard() + _write_scorecard(report_dir, scorecard) + + store(report_dir, db_url, "tekton-run-004") + store(report_dir, db_url, "tekton-run-004") + + with session_factory() as session: + scorecards = session.execute(select(ScorecardRow)).scalars().all() + assert len(scorecards) == 1 diff --git a/uv.lock b/uv.lock index a468619..f43c9c4 100644 --- a/uv.lock +++ b/uv.lock @@ -15,6 +15,7 @@ name = "abevalflow" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "alembic" }, { name = "jinja2" }, { name = "kubernetes" }, { name = "matplotlib" }, @@ -39,6 +40,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "alembic", specifier = ">=1.13" }, { name = "jinja2", specifier = ">=3.1" }, { name = "kubernetes", specifier = ">=28.0" }, { name = "matplotlib", specifier = ">=3.8" }, @@ -58,6 +60,20 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "alembic" +version = "1.18.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/cc/ac0bed8e562e7407fe55c3ba85a4dce86e6dbd8730887bd1e406a6c5c18a/alembic-1.18.5.tar.gz", hash = "sha256:1554982221dd17e9a749b53902407578eb305e453f71999e8c7f0a48389fff8e", size = 2060480, upload-time = "2026-06-25T15:20:54.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/78/5fe6dc3a3a5b2f5a2a4faef8bfe336d5fa049a38884ab3172e0098160c01/alembic-1.18.5-py3-none-any.whl", hash = "sha256:06d8ba9d04558022f5395e9317de03d270f3dced49cee01f89fe7a13c26f14bc", size = 264664, upload-time = "2026-06-25T15:20:56.673Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -571,9 +587,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" }, { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" }, { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" }, - { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" }, { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" }, - { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" }, { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" }, { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" }, { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" }, @@ -581,9 +595,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" }, { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" }, { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" }, - { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" }, { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" }, - { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" }, { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" }, { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" }, { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" }, @@ -591,9 +603,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, - { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" }, { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, - { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" }, { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, @@ -601,9 +611,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" }, { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" }, { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" }, - { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" }, { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" }, - { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" }, { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" }, { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" }, { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" }, @@ -611,9 +619,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" }, { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" }, { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" }, - { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" }, { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" }, - { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" }, { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" }, { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" }, { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" }, @@ -902,6 +908,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/70/05b685ea2dffcb2adbf3cdcea5d8865b7bc66f67249084cf845012a0ff13/kubernetes-35.0.0-py2.py3-none-any.whl", hash = "sha256:39e2b33b46e5834ef6c3985ebfe2047ab39135d41de51ce7641a7ca5b372a13d", size = 2017602, upload-time = "2026-01-16T01:05:25.991Z" }, ] +[[package]] +name = "mako" +version = "1.3.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/62/791b31e69ae182791ec67f04850f2f062716bbd205483d63a215f3e062d3/mako-1.3.12.tar.gz", hash = "sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a", size = 400219, upload-time = "2026-04-28T19:01:08.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/b1/a0ec7a5a9db730a08daef1fdfb8090435b82465abbf758a596f0ea88727e/mako-1.3.12-py3-none-any.whl", hash = "sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9", size = 78521, upload-time = "2026-04-28T19:01:10.393Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" From 60e734029fc2ae686b45893c4061196adf8265b3 Mon Sep 17 00:00:00 2001 From: apetruni Date: Wed, 8 Jul 2026 14:33:29 +0300 Subject: [PATCH 2/4] fix: add pyyaml to store task dependencies The new Scorecard import in store_results.py transitively imports certification.py which requires PyYAML. Co-Authored-By: Claude Opus 4.6 (1M context) --- pipeline/tasks/post/store.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/tasks/post/store.yaml b/pipeline/tasks/post/store.yaml index 1a9af60..0ea39da 100644 --- a/pipeline/tasks/post/store.yaml +++ b/pipeline/tasks/post/store.yaml @@ -124,7 +124,7 @@ spec: export PYTHONPATH="$PIPELINE_DIR" # --- 2. Install dependencies --- - pip install --quiet --no-cache-dir pydantic sqlalchemy "psycopg[binary]" "tenacity>=8.2" + pip install --quiet --no-cache-dir pydantic sqlalchemy "psycopg[binary]" "tenacity>=8.2" pyyaml # --- 3. Run store script --- REPORT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" From c83f4daff24ffff2e8dbe82b5578a234baf0ed92 Mon Sep 17 00:00:00 2001 From: apetruni Date: Wed, 8 Jul 2026 15:39:59 +0300 Subject: [PATCH 3/4] fix: remove unused imports flagged by ruff Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/backfill_scorecards.py | 3 +-- tests/test_observability_models.py | 1 - tests/test_store_results_observability.py | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/backfill_scorecards.py b/scripts/backfill_scorecards.py index 745b0c8..e54d3e4 100644 --- a/scripts/backfill_scorecards.py +++ b/scripts/backfill_scorecards.py @@ -28,7 +28,6 @@ from __future__ import annotations import argparse -import io import json import logging import os @@ -39,7 +38,7 @@ from sqlalchemy import select from abevalflow.db.engine import get_engine, init_db, make_session -from abevalflow.db.models import CertificationRow, GateResultRow, ScorecardRow +from abevalflow.db.models import ScorecardRow from abevalflow.scorecard import Scorecard from scripts.store_results import map_certifications, map_gate_results, map_scorecard_to_row diff --git a/tests/test_observability_models.py b/tests/test_observability_models.py index 64c61f7..56d9107 100644 --- a/tests/test_observability_models.py +++ b/tests/test_observability_models.py @@ -4,7 +4,6 @@ import uuid from datetime import UTC, datetime -from decimal import Decimal import pytest from sqlalchemy import create_engine, select diff --git a/tests/test_store_results_observability.py b/tests/test_store_results_observability.py index 4db9c99..b6639f7 100644 --- a/tests/test_store_results_observability.py +++ b/tests/test_store_results_observability.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from pathlib import Path import pytest @@ -11,7 +10,6 @@ from abevalflow.db.models import ( Base, - CertificationRow, EvaluationRun, GateResultRow, ScorecardRow, From 5000ab4a7980adcd82d3566bb5e05798bc9526df Mon Sep 17 00:00:00 2001 From: apetruni Date: Wed, 8 Jul 2026 15:42:48 +0300 Subject: [PATCH 4/4] style: apply ruff formatting Co-Authored-By: Claude Opus 4.6 (1M context) --- abevalflow/db/models.py | 4 +--- scripts/backfill_scorecards.py | 27 ++++++++++++----------- tests/test_store_results_observability.py | 25 ++++++++++++++++----- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/abevalflow/db/models.py b/abevalflow/db/models.py index 6a96a8b..5ae8f0f 100644 --- a/abevalflow/db/models.py +++ b/abevalflow/db/models.py @@ -287,9 +287,7 @@ class ScorecardRow(Base): created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow) - gate_results: Mapped[list[GateResultRow]] = relationship( - back_populates="scorecard", cascade="all, delete-orphan" - ) + gate_results: Mapped[list[GateResultRow]] = relationship(back_populates="scorecard", cascade="all, delete-orphan") certifications: Mapped[list[CertificationRow]] = relationship( back_populates="scorecard", cascade="all, delete-orphan" ) diff --git a/scripts/backfill_scorecards.py b/scripts/backfill_scorecards.py index e54d3e4..6ac62f6 100644 --- a/scripts/backfill_scorecards.py +++ b/scripts/backfill_scorecards.py @@ -68,13 +68,18 @@ def _load_checkpoint(path: Path) -> BackfillState: def _save_checkpoint(path: Path, state: BackfillState) -> None: - path.write_text(json.dumps({ - "last_processed_key": state.last_processed_key, - "processed_count": state.processed_count, - "skipped_count": state.skipped_count, - "error_count": state.error_count, - "errors": state.errors[-50:], - }, indent=2)) + path.write_text( + json.dumps( + { + "last_processed_key": state.last_processed_key, + "processed_count": state.processed_count, + "skipped_count": state.skipped_count, + "error_count": state.error_count, + "errors": state.errors[-50:], + }, + indent=2, + ) + ) def backfill( @@ -111,9 +116,7 @@ def backfill( objects = client.list_objects(bucket, prefix=prefix_filter, recursive=True) scorecard_keys = sorted( - obj.object_name - for obj in objects - if obj.object_name and obj.object_name.endswith("/scorecard.json") + obj.object_name for obj in objects if obj.object_name and obj.object_name.endswith("/scorecard.json") ) if resume and state.last_processed_key: @@ -140,9 +143,7 @@ def backfill( with session_factory() as session: existing = session.execute( - select(ScorecardRow).where( - ScorecardRow.pipeline_run_id == scorecard.pipeline_run_id - ) + select(ScorecardRow).where(ScorecardRow.pipeline_run_id == scorecard.pipeline_run_id) ).scalar_one_or_none() if existing is not None: diff --git a/tests/test_store_results_observability.py b/tests/test_store_results_observability.py index b6639f7..4e36dda 100644 --- a/tests/test_store_results_observability.py +++ b/tests/test_store_results_observability.py @@ -46,14 +46,29 @@ def _sample_result(name: str = "my-submission") -> AnalysisResult: ), summary=AnalysisSummary( treatment=VariantSummary( - n_trials=20, n_passed=16, n_failed=3, n_errors=1, - pass_rate=0.8, mean_reward=0.72, median_reward=0.75, std_reward=0.15, + n_trials=20, + n_passed=16, + n_failed=3, + n_errors=1, + pass_rate=0.8, + mean_reward=0.72, + median_reward=0.75, + std_reward=0.15, ), control=VariantSummary( - n_trials=20, n_passed=10, n_failed=8, n_errors=2, - pass_rate=0.5, mean_reward=0.45, median_reward=0.42, std_reward=0.20, + n_trials=20, + n_passed=10, + n_failed=8, + n_errors=2, + pass_rate=0.5, + mean_reward=0.45, + median_reward=0.42, + std_reward=0.20, ), - uplift=0.3, mean_reward_gap=0.27, ttest_p_value=0.02, fisher_p_value=0.04, + uplift=0.3, + mean_reward_gap=0.27, + ttest_p_value=0.02, + fisher_p_value=0.04, recommendation=Recommendation.PASS, ), trials={