fix(validator): guard legacy ensure_schema alters for fresh DBs (#120)

docs/JOURNAL.md

@@ -18,6 +18,16 @@ protocol. **Newest entries at the top.** Tag each entry with one or more of:
 
 ---
 
+## 2026-07-30 — ensure_schema still crashed on fresh DBs after lock-timeout helpers  #mistake #repro #decision
+**Context:** issue #120; main added `_drop_not_null_if_needed` / `_column_nullable` for lock avoidance.
+**Expected:** fresh `create_all` + `ensure_schema` succeeds.
+**Actual:** `_drop_not_null_if_needed` treated a missing column as "not nullable" and still issued
+`ALTER COLUMN … DROP NOT NULL`; legacy `UPDATE … team_name` / `benchmark` also ran unconditionally.
+**Root cause:** existence was not checked before ALTER/UPDATE of legacy columns.
+**Fix / decision:** require `_column_exists` before DROP NOT NULL; gate legacy back-fills the same way.
+Added `validator/tests/test_ensure_schema.py`.
+**Follow-up:** none.
+
 ## 2026-07-12 — code grader: add resource limits on top of the HOME/secrets fix  #security #decision
 **Context:** issue #71 — the code grader (`run_pass_at_1`) runs untrusted miner/LLM candidate code. The core secret-leak fix (isolated throwaway HOME/cwd, scrubbed env, `python -I`) already landed on main.
 **Finding:** main's sandbox closes the HOME/secrets exfiltration vector but has **no resource limits** — an untrusted candidate can still exhaust host memory or fork-bomb the eval box within its wall-clock timeout (verified: a 4 GiB `bytearray` allocation runs to completion and "passes" on main).

validator/src/eval_backend/db.py

@@ -62,7 +62,11 @@ def _add_column_if_missing(conn, table: str, column: str, ddl_type: str) -> None
 
 
 def _drop_not_null_if_needed(conn, table: str, column: str) -> None:
-    if not _column_nullable(conn, table, column):
+    # Missing columns must be skipped: on a fresh DB the legacy submission
+    # fields are ORM @property accessors, not real columns (issue #120).
+    # `_column_nullable` returns False when the column is absent, so without
+    # an existence check we would still issue ALTER COLUMN and crash.
+    if _column_exists(conn, table, column) and not _column_nullable(conn, table, column):
         conn.exec_driver_sql(f"ALTER TABLE {table} ALTER COLUMN {column} DROP NOT NULL")
 
 
@@ -101,14 +105,17 @@ def ensure_schema(engine) -> None:
         _drop_not_null_if_needed(conn, "submissions", "artifact_sha256")
         _drop_not_null_if_needed(conn, "submissions", "checkpoint_path")
         _drop_not_null_if_needed(conn, "submissions", "benchmark")
-        conn.exec_driver_sql(
-            "UPDATE submissions SET miner_id = COALESCE(miner_id, team_name) "
-            "WHERE miner_id IS NULL AND team_name IS NOT NULL"
-        )
-        conn.exec_driver_sql(
-            "UPDATE submissions SET benchmark_names_json = COALESCE(benchmark_names_json, json_build_array(benchmark)) "
-            "WHERE benchmark_names_json IS NULL AND benchmark IS NOT NULL"
-        )
+        # Legacy back-fills reference columns that only exist on pre-property DBs.
+        if _column_exists(conn, "submissions", "team_name"):
+            conn.exec_driver_sql(
+                "UPDATE submissions SET miner_id = COALESCE(miner_id, team_name) "
+                "WHERE miner_id IS NULL AND team_name IS NOT NULL"
+            )
+        if _column_exists(conn, "submissions", "benchmark"):
+            conn.exec_driver_sql(
+                "UPDATE submissions SET benchmark_names_json = COALESCE(benchmark_names_json, json_build_array(benchmark)) "
+                "WHERE benchmark_names_json IS NULL AND benchmark IS NOT NULL"
+            )
         conn.exec_driver_sql(
             "UPDATE competition_runtime_config "
             "SET default_eval_execution_mode = COALESCE(NULLIF(default_eval_execution_mode, ''), 'remote_gpu')"

validator/tests/test_ensure_schema.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from sqlalchemy import text
+
+from eval_backend.db import Base, ensure_schema
+import eval_backend.models  # noqa: F401  — register ORM metadata
+
+_LEGACY_COLUMNS = (
+    "artifact_name",
+    "artifact_path",
+    "artifact_sha256",
+    "checkpoint_path",
+    "benchmark",
+    "team_name",
+)
+
+
+def test_ensure_schema_succeeds_on_fresh_create_all(validator_engine) -> None:
+    """Fresh DBs have no legacy submission columns; ensure_schema must not crash."""
+    Base.metadata.create_all(validator_engine)
+    ensure_schema(validator_engine)
+
+    with validator_engine.connect() as conn:
+        columns = {
+            row[0]
+            for row in conn.execute(
+                text(
+                    "SELECT column_name FROM information_schema.columns "
+                    "WHERE table_name = 'submissions' AND table_schema = current_schema()"
+                )
+            )
+        }
+    assert "miner_id" in columns
+    assert "benchmark_names_json" in columns
+    assert "artifact_name" not in columns
+    assert "team_name" not in columns
+
+
+def test_ensure_schema_relaxes_and_backfills_legacy_columns(validator_engine) -> None:
+    """When legacy columns exist, ensure_schema should relax NOT NULL and back-fill."""
+    Base.metadata.create_all(validator_engine)
+
+    try:
+        with validator_engine.begin() as conn:
+            for column, coltype in (
+                ("artifact_name", "VARCHAR(255)"),
+                ("artifact_path", "VARCHAR(1024)"),
+                ("artifact_sha256", "VARCHAR(64)"),
+                ("checkpoint_path", "VARCHAR(1024)"),
+                ("benchmark", "VARCHAR(64)"),
+                ("team_name", "VARCHAR(255)"),
+            ):
+                conn.execute(
+                    text(f"ALTER TABLE submissions ADD COLUMN IF NOT EXISTS {column} {coltype}")
+                )
+                conn.execute(text(f"UPDATE submissions SET {column} = '' WHERE {column} IS NULL"))
+                conn.execute(text(f"ALTER TABLE submissions ALTER COLUMN {column} SET NOT NULL"))
+
+            conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'"))
+            conn.execute(
+                text(
+                    """
+                    INSERT INTO submissions (
+                        id, source, team_name, artifact_name, artifact_path,
+                        artifact_sha256, checkpoint_path, benchmark, status,
+                        benchmark_names_json, created_at, updated_at
+                    ) VALUES (
+                        'legacy-1', 'upload', 'old-team', 'a.npy', '/tmp/a.npy',
+                        'deadbeef', '/tmp/ckpt.npy', 'math500', 'queued',
+                        '[]'::json, NOW(), NOW()
+                    )
+                    """
+                )
+            )
+            conn.execute(
+                text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json DROP NOT NULL")
+            )
+            conn.execute(
+                text("UPDATE submissions SET benchmark_names_json = NULL WHERE id = 'legacy-1'")
+            )
+            conn.execute(text("UPDATE submissions SET miner_id = NULL WHERE id = 'legacy-1'"))
+
+        ensure_schema(validator_engine)
+
+        with validator_engine.connect() as conn:
+            row = conn.execute(
+                text(
+                    "SELECT miner_id, benchmark_names_json::text, "
+                    "(SELECT is_nullable FROM information_schema.columns "
+                    " WHERE table_name = 'submissions' AND column_name = 'artifact_name' "
+                    "   AND table_schema = current_schema()) "
+                    "FROM submissions WHERE id = 'legacy-1'"
+                )
+            ).one()
+        assert row[0] == "old-team"
+        assert "math500" in (row[1] or "")
+        assert row[2] == "YES"
+    finally:
+        with validator_engine.begin() as conn:
+            for column in _LEGACY_COLUMNS:
+                conn.execute(text(f"ALTER TABLE submissions DROP COLUMN IF EXISTS {column}"))
+            conn.execute(
+                text("ALTER TABLE submissions ALTER COLUMN benchmark_names_json SET NOT NULL")
+            )
+            conn.execute(text("DELETE FROM submissions WHERE id = 'legacy-1'"))