From 9bb836511068e6382a9cbbdb49e7a062305b43a3 Mon Sep 17 00:00:00 2001 From: tommywsh9 Date: Tue, 12 May 2026 10:05:22 +0000 Subject: [PATCH 1/2] v0.3.0: Phase 2 complete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Scenario library — 4 presets (CIFAR-10 non-IID, MNIST IID, CIFAR-10 high-malicious, CIFAR-100), per-scenario leaderboards and matrix filtering - Multi-metric scoring — Accuracy Drop, Convergence Speed, Stability, Runtime, per-opponent and summary aggregation - Multi-dimensional leaderboard — sortable columns, baseline merge, scenario selector - Dashboard page — live arena stats, active jobs, top methods, recent submissions - Sidebar navigation — Dashboard, Arena, Bench, Leaderboard, Compare, Jobs - Matrix interaction — click cell for per-seed breakdown and stats - Arena UX — 4 template cards, evaluation intensity selector (Quick/Standard/Full) - Experiment comparison — /compare page with multi-select, side-by-side metrics and overlay charts - Method versioning — same method iteration tracking, auto-suffix method_name for FL registry uniqueness, version history panel, leaderboard dedup - LLM analytical reports — cached per evaluation, regenerate on demand - Training curve visualization — per-opponent accuracy/loss charts - 53 backend tests, all passing --- README.md | 28 +- README.zh-CN.md | 7 +- apps/backend/app/api/v1/bench.py | 2 + apps/backend/app/api/v1/dashboard.py | 135 ++++++ apps/backend/app/api/v1/jobs.py | 63 ++- apps/backend/app/api/v1/leaderboard.py | 72 +++- apps/backend/app/api/v1/matrix.py | 19 +- apps/backend/app/api/v1/router.py | 4 + apps/backend/app/api/v1/scenarios.py | 25 ++ apps/backend/app/api/v1/submissions.py | 152 ++++++- apps/backend/app/main.py | 29 +- apps/backend/app/models.py | 6 + apps/backend/app/schemas.py | 39 ++ apps/backend/app/services/analysis.py | 148 +++++++ apps/backend/app/services/evaluation.py | 305 +++++++++---- apps/backend/app/services/report.py | 52 ++- apps/backend/app/services/submission.py | 16 + apps/backend/tests/test_dashboard.py | 87 ++++ apps/backend/tests/test_submissions.py | 5 +- apps/frontend/src/App.tsx | 163 +++++-- apps/frontend/src/api/client.ts | 131 +++++- .../src/components/IntensitySelector.tsx | 39 ++ .../src/components/ScenarioSelector.tsx | 32 ++ apps/frontend/src/context/AgentContext.tsx | 93 ++++ apps/frontend/src/context/BenchContext.tsx | 75 ++++ apps/frontend/src/pages/Agent.tsx | 137 +++--- apps/frontend/src/pages/Bench.tsx | 118 ++--- apps/frontend/src/pages/BenchDetail.tsx | 185 ++++++++ apps/frontend/src/pages/Compare.tsx | 402 ++++++++++++++++++ apps/frontend/src/pages/Dashboard.tsx | 178 ++++++++ apps/frontend/src/pages/Detail.tsx | 310 ++++++++++---- apps/frontend/src/pages/Jobs.tsx | 158 +++++++ apps/frontend/src/pages/Leaderboard.tsx | 106 +++-- apps/frontend/src/pages/Matrix.tsx | 120 +++++- apps/frontend/src/pages/Submit.tsx | 208 ++++++++- apps/frontend/src/utils/exportPdf.ts | 24 +- .../research/scenarios/cifar100_noniid.yaml | 52 +++ .../scenarios/cifar10_high_malicious.yaml | 52 +++ .../research/scenarios/cifar10_noniid.yaml | 52 +++ configs/research/scenarios/mnist_iid.yaml | 52 +++ libs/fl_core/research/arena.py | 61 ++- libs/fl_core/research/runner.py | 37 +- libs/fl_core/research/scenarios.py | 77 ++++ 43 files changed, 3600 insertions(+), 456 deletions(-) create mode 100644 apps/backend/app/api/v1/dashboard.py create mode 100644 apps/backend/app/api/v1/scenarios.py create mode 100644 apps/backend/app/services/analysis.py create mode 100644 apps/backend/tests/test_dashboard.py create mode 100644 apps/frontend/src/components/IntensitySelector.tsx create mode 100644 apps/frontend/src/components/ScenarioSelector.tsx create mode 100644 apps/frontend/src/context/AgentContext.tsx create mode 100644 apps/frontend/src/context/BenchContext.tsx create mode 100644 apps/frontend/src/pages/BenchDetail.tsx create mode 100644 apps/frontend/src/pages/Compare.tsx create mode 100644 apps/frontend/src/pages/Dashboard.tsx create mode 100644 apps/frontend/src/pages/Jobs.tsx create mode 100644 configs/research/scenarios/cifar100_noniid.yaml create mode 100644 configs/research/scenarios/cifar10_high_malicious.yaml create mode 100644 configs/research/scenarios/cifar10_noniid.yaml create mode 100644 configs/research/scenarios/mnist_iid.yaml create mode 100644 libs/fl_core/research/scenarios.py diff --git a/README.md b/README.md index cc299c7..0ddfe9f 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ class MyAttack(ResearchAttackStrategy): ``` ┌──────────────────────────────────────────────────────────────┐ │ React + Vite Frontend │ -│ (Leaderboard · Arena · Bench · Detail) │ +│ (Dashboard · Arena · Bench · Leaderboard · Jobs · Detail) │ └───────────────────────────┬──────────────────────────────────┘ │ REST + polling ┌───────────────────────────▼──────────────────────────────────┐ @@ -201,7 +201,7 @@ fedarena/ │ │ │ └── config.py # Pydantic settings (.env loading) │ │ └── runners/ # FL runtime (core_runtime.py) │ └── frontend/ # React + Vite + Tailwind + Radix UI -│ └── src/pages/ # Leaderboard, Arena, Bench, Detail +│ └── src/pages/ # Dashboard, Arena, Bench, Leaderboard, Jobs, Detail ├── libs/fl_core/ # FL core library │ ├── research/ # Arena engine (registry, runner, arena, base classes) │ │ ├── attacks/ # Baseline + user submissions @@ -234,21 +234,21 @@ fedarena/ ### Phase 2: Range / Scenario System -- [ ] Scenario library — multiple datasets, non-IID levels, malicious client ratios, model architectures -- [ ] Multi-metric scoring — Attack Success Rate, Accuracy Drop, Convergence Speed, Runtime Cost, confidence intervals -- [ ] Multi-dimensional leaderboard — Overall Score, Effectiveness, Robustness, Stability, Runtime columns -- [ ] Per-scenario leaderboards -- [ ] Sandbox execution — container isolation, timeout, network & filesystem restrictions for user-submitted code -- [ ] Method versioning — track iterations of the same attack/defense, side-by-side comparison -- [ ] Experiment comparison — cross-run overlay charts, statistical significance tests -- [ ] Analytical reports — LLM-generated strengths/weaknesses, failure analysis, baseline diff, next-step recommendations -- [ ] Dashboard page — live arena status, active jobs, top methods, recent submissions -- [ ] Navigation restructure — Dashboard / Scenarios / Arena / Bench / Leaderboard / Reports / Methods / Jobs -- [ ] Arena UX improvements — left/right layout, template mode, evaluation intensity selection -- [ ] Matrix interaction — filter by scenario, click cell for run details & training curves +- [x] Scenario library — multiple datasets, non-IID levels, malicious client ratios, model architectures +- [x] Multi-metric scoring — Accuracy Drop, Convergence Speed, Stability, Runtime Cost, Max Accuracy, per-opponent and summary aggregation +- [x] Multi-dimensional leaderboard — Avg Accuracy, Accuracy Drop, Worst Case, Convergence, Stability columns with sort_by support +- [x] Per-scenario leaderboards +- [x] Method versioning — track iterations of the same attack/defense, side-by-side comparison +- [x] Experiment comparison — cross-run overlay charts, side-by-side metrics +- [x] Analytical reports — LLM-generated strengths/weaknesses, baseline comparison, recommendations, cached per evaluation +- [x] Dashboard page — live arena status, active jobs, top methods, recent submissions +- [x] Navigation restructure — Dashboard / Scenarios / Arena / Bench / Leaderboard / Reports / Methods / Jobs +- [x] Arena UX improvements — left/right layout, template mode, evaluation intensity selection +- [x] Matrix interaction — filter by scenario, click cell for run details & per-seed breakdown ### Phase 3: Platform +- [ ] Sandbox execution — container isolation, timeout, network & filesystem restrictions for user-submitted code - [ ] User & team accounts with permissions - [ ] Challenge mode — fixed scenarios, time-limited competitions, hidden test sets - [ ] Course mode — guided exercises for FL security education diff --git a/README.zh-CN.md b/README.zh-CN.md index 9370653..074d9c4 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -209,13 +209,12 @@ fedarena/ 欢迎贡献！FedArena 致力于成为简洁、可扩展的联邦学习安全研究平台。 -- [ ] **多配置矩阵** — 支持多数据集 / IID / 客户端数配置，拓宽评测覆盖面 -- [ ] **Markdown 报告生成** — 可导出的逐提交对比报告 +- [x] **多配置矩阵** — 场景库：多数据集 / non-IID 级别 / 恶意比例 / 模型架构 +- [x] **Markdown 报告生成** — 可导出的逐提交对比报告（Markdown + PDF） - [ ] **用户认证** — 简单的账号系统，关联提交归属 - [ ] **Docker 部署** — 一键 `docker-compose` 启动后端 + 前端 + GPU -- [ ] **更多攻防方法** — 数据投毒、自适应裁剪、几何中位数等 - [ ] **可复现性保障** — 固定种子、配置哈希校验、环境指纹 -- [ ] **CI 流水线** — 自动化测试：注册表发现、提交校验、API 端点 +- [x] **CI 流水线** — 自动化测试：注册表发现、提交校验、API 端点

FedArena 仅供研究和教学用途。 diff --git a/apps/backend/app/api/v1/bench.py b/apps/backend/app/api/v1/bench.py index 2f3872c..e680311 100644 --- a/apps/backend/app/api/v1/bench.py +++ b/apps/backend/app/api/v1/bench.py @@ -42,6 +42,7 @@ class BenchJobResponse(BaseModel): error: str | None started_at: str | None completed_at: str | None + created_at: str | None def _build_response(job: BenchJob) -> BenchJobResponse: @@ -64,6 +65,7 @@ def _build_response(job: BenchJob) -> BenchJobResponse: error=job.error, started_at=str(job.started_at) if job.started_at else None, completed_at=str(job.completed_at) if job.completed_at else None, + created_at=str(job.bench_created_at), ) diff --git a/apps/backend/app/api/v1/dashboard.py b/apps/backend/app/api/v1/dashboard.py new file mode 100644 index 0000000..e556c71 --- /dev/null +++ b/apps/backend/app/api/v1/dashboard.py @@ -0,0 +1,135 @@ +"""Dashboard aggregation endpoint.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession +from sqlmodel import func, select + +from ...db import get_session +from ...models import BenchJob, EvaluationJob, Submission +from ...services.task_queue import get_queue + +router = APIRouter(prefix="/dashboard", tags=["dashboard"]) + + +class TopMethod(BaseModel): + submission_id: int + display_name: str + method_name: str + author: str | None + avg_accuracy: float + + +class RecentSubmission(BaseModel): + id: int + display_name: str + method_name: str + role: str + status: str + created_at: str + + model_config = {"from_attributes": True} + + +class DashboardResponse(BaseModel): + total_submissions: int + completed_evaluations: int + failed_evaluations: int + active_jobs: int + queue_pending: int + top_attack: TopMethod | None + top_defense: TopMethod | None + recent_submissions: list[RecentSubmission] + + +@router.get("", response_model=DashboardResponse) +async def get_dashboard(session: AsyncSession = Depends(get_session)): + total_submissions = (await session.execute(select(func.count(Submission.id)))).scalar() or 0 + + completed_evaluations = ( + await session.execute(select(func.count(EvaluationJob.id)).where(EvaluationJob.status == "completed")) + ).scalar() or 0 + + failed_evaluations = ( + await session.execute(select(func.count(EvaluationJob.id)).where(EvaluationJob.status == "failed")) + ).scalar() or 0 + + active_eval = ( + await session.execute( + select(func.count(EvaluationJob.id)).where(EvaluationJob.status.in_(["running", "queued"])) + ) + ).scalar() or 0 + + active_bench = ( + await session.execute(select(func.count(BenchJob.id)).where(BenchJob.status.in_(["running", "queued"]))) + ).scalar() or 0 + + queue_pending = get_queue().pending_count() + + top_attack = await _top_method(session, "attack") + top_defense = await _top_method(session, "defense") + + recent_stmt = select(Submission).order_by(Submission.created_at.desc(), Submission.id.desc()).limit(8) + recent_rows = (await session.execute(recent_stmt)).scalars().all() + recent_submissions = [ + RecentSubmission( + id=s.id, + display_name=s.display_name, + method_name=s.method_name, + role=s.role, + status=s.status, + created_at=str(s.created_at), + ) + for s in recent_rows + ] + + return DashboardResponse( + total_submissions=total_submissions, + completed_evaluations=completed_evaluations, + failed_evaluations=failed_evaluations, + active_jobs=active_eval + active_bench, + queue_pending=queue_pending, + top_attack=top_attack, + top_defense=top_defense, + recent_submissions=recent_submissions, + ) + + +async def _top_method(session: AsyncSession, role: str) -> TopMethod | None: + import json + + stmt = ( + select(Submission, EvaluationJob) + .join(EvaluationJob, EvaluationJob.submission_id == Submission.id) + .where(Submission.role == role, Submission.status == "completed", EvaluationJob.status == "completed") + .order_by(EvaluationJob.completed_at.desc()) + ) + rows = (await session.execute(stmt)).all() + + best: TopMethod | None = None + best_score: float | None = None + + for sub, job in rows: + if not job.results_json: + continue + results = json.loads(job.results_json) + accs = [r["avg_final_accuracy"] for r in results.values() if r.get("avg_final_accuracy") is not None] + if not accs: + continue + avg = sum(accs) / len(accs) + is_better = ( + best_score is None or (role == "attack" and avg < best_score) or (role == "defense" and avg > best_score) + ) + if is_better: + best_score = avg + best = TopMethod( + submission_id=sub.id, + display_name=sub.display_name, + method_name=sub.method_name, + author=sub.author, + avg_accuracy=avg, + ) + + return best diff --git a/apps/backend/app/api/v1/jobs.py b/apps/backend/app/api/v1/jobs.py index 31bb2f4..f164d68 100644 --- a/apps/backend/app/api/v1/jobs.py +++ b/apps/backend/app/api/v1/jobs.py @@ -5,9 +5,10 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession +from sqlmodel import select from ...db import get_session -from ...models import EvaluationJob +from ...models import BenchJob, EvaluationJob, Submission from ...schemas import JobProgress from ...services.task_queue import get_queue @@ -21,6 +22,66 @@ class QueueStatusResponse(BaseModel): tasks: list[dict] +class JobListItem(BaseModel): + id: int + job_type: str # "evaluation" | "bench" + status: str + label: str + submission_id: int | None = None + progress: str | None = None + error: str | None = None + started_at: str | None = None + completed_at: str | None = None + created_at: str + + +@router.get("", response_model=list[JobListItem]) +async def list_jobs(session: AsyncSession = Depends(get_session)): + eval_stmt = ( + select(EvaluationJob, Submission.display_name) + .join(Submission, Submission.id == EvaluationJob.submission_id, isouter=True) + .order_by(EvaluationJob.created_at.desc()) + .limit(50) + ) + eval_rows = (await session.execute(eval_stmt)).all() + + bench_stmt = select(BenchJob).order_by(BenchJob.bench_created_at.desc()).limit(50) + bench_rows = (await session.execute(bench_stmt)).scalars().all() + + items: list[JobListItem] = [] + for job, display_name in eval_rows: + items.append( + JobListItem( + id=job.id, + job_type="evaluation", + status=job.status, + label=display_name or f"Submission #{job.submission_id}", + submission_id=job.submission_id, + progress=job.progress, + error=job.error, + started_at=str(job.started_at) if job.started_at else None, + completed_at=str(job.completed_at) if job.completed_at else None, + created_at=str(job.created_at), + ) + ) + for job in bench_rows: + items.append( + JobListItem( + id=job.id, + job_type="bench", + status=job.status, + label=job.prompt[:80] if job.prompt else f"Bench #{job.id}", + error=job.error, + started_at=str(job.started_at) if job.started_at else None, + completed_at=str(job.completed_at) if job.completed_at else None, + created_at=str(job.bench_created_at), + ) + ) + + items.sort(key=lambda x: x.created_at, reverse=True) + return items[:50] + + @router.get("/queue/status", response_model=QueueStatusResponse) async def queue_status(): """Get current task queue status.""" diff --git a/apps/backend/app/api/v1/leaderboard.py b/apps/backend/app/api/v1/leaderboard.py index d07c8f4..5f3e7e5 100644 --- a/apps/backend/app/api/v1/leaderboard.py +++ b/apps/backend/app/api/v1/leaderboard.py @@ -15,9 +15,26 @@ router = APIRouter(prefix="/leaderboard", tags=["leaderboard"]) +SORT_KEYS = {"avg_accuracy", "avg_accuracy_drop", "worst_case_accuracy", "avg_convergence_speed", "avg_stability"} + + +def _extract_scenario_results(results_json: str, scenario: str | None) -> dict | None: + """Extract results for a specific scenario, handling both old and new formats.""" + results = json.loads(results_json) + if "scenarios" in results: + target = scenario or "cifar10_noniid" + return results["scenarios"].get(target) + if scenario is None or scenario == "cifar10_noniid": + return results + return None + + @router.get("", response_model=list[LeaderboardEntry]) async def get_leaderboard( role: str = Query(..., pattern="^(attack|defense)$"), + sort_by: str = Query("avg_accuracy"), + scenario: str | None = Query(None), + show_all_versions: bool = Query(False), session: AsyncSession = Depends(get_session), ): """Get ranked leaderboard for attacks or defenses. @@ -25,6 +42,9 @@ async def get_leaderboard( Attacks ranked by lowest avg accuracy (stronger attack = lower accuracy). Defenses ranked by highest avg accuracy (stronger defense = higher accuracy). """ + if sort_by not in SORT_KEYS: + sort_by = "avg_accuracy" + stmt = ( select(Submission, EvaluationJob) .join(EvaluationJob, EvaluationJob.submission_id == Submission.id) @@ -33,16 +53,36 @@ async def get_leaderboard( ) rows = (await session.execute(stmt)).all() - # Deduplicate: keep latest job per submission seen: dict[int, tuple] = {} for sub, job in rows: if sub.id not in seen and job.results_json: seen[sub.id] = (sub, job) - # Parse results and compute avg accuracy entries: list[dict] = [] + group_versions: dict[str, list[int]] = {} for sub, job in seen.values(): - results = json.loads(job.results_json) + group = sub.method_group or sub.method_name + ver = sub.version or 1 + group_versions.setdefault(group, []).append(ver) + + # When deduplicating, keep only the highest version per group + if not show_all_versions: + best_per_group: dict[str, int] = {} + for sub, job in seen.values(): + group = sub.method_group or sub.method_name + ver = sub.version or 1 + if group not in best_per_group or ver > best_per_group[group]: + best_per_group[group] = ver + + for sub, job in seen.values(): + group = sub.method_group or sub.method_name + ver = sub.version or 1 + if not show_all_versions and best_per_group.get(group) != ver: + continue + results = _extract_scenario_results(job.results_json, scenario) + if results is None: + continue + summary = results.get("__summary__", {}) accs = [v["avg_final_accuracy"] for v in results.values() if v.get("avg_final_accuracy") is not None] if not accs: continue @@ -55,14 +95,30 @@ async def get_leaderboard( "author": sub.author, "role": sub.role, "avg_accuracy": avg, - "opponent_scores": {k: v.get("avg_final_accuracy") for k, v in results.items()}, + "opponent_scores": { + k: v.get("avg_final_accuracy") for k, v in results.items() if k != "__summary__" + }, "submitted_at": sub.created_at, + "avg_accuracy_drop": summary.get("avg_accuracy_drop"), + "worst_case_accuracy": summary.get("worst_case_accuracy"), + "avg_convergence_speed": summary.get("avg_convergence_speed"), + "avg_stability": summary.get("avg_stability"), + "version": ver, + "has_older_versions": len(group_versions.get(group, [])) > 1, } ) - # Sort: attacks ascending (lower = stronger), defenses descending - reverse = role == "defense" - entries.sort(key=lambda e: e["avg_accuracy"], reverse=reverse) + if sort_by == "avg_accuracy": + reverse = role == "defense" + elif sort_by in ("avg_accuracy_drop",): + reverse = role == "attack" + elif sort_by == "avg_convergence_speed": + reverse = False + elif sort_by == "avg_stability": + reverse = False + else: + reverse = role == "defense" + + entries.sort(key=lambda e: e.get(sort_by) or 0, reverse=reverse) - # Assign ranks return [LeaderboardEntry(rank=i + 1, **entry) for i, entry in enumerate(entries)] diff --git a/apps/backend/app/api/v1/matrix.py b/apps/backend/app/api/v1/matrix.py index a689592..3086bf5 100644 --- a/apps/backend/app/api/v1/matrix.py +++ b/apps/backend/app/api/v1/matrix.py @@ -3,30 +3,31 @@ from __future__ import annotations import json -from pathlib import Path -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Query from ...schemas import MatrixResponse router = APIRouter(prefix="/matrix", tags=["matrix"]) -MATRIX_PATH = Path(__file__).resolve().parents[5] / "results" / "arena" / "benchmark_matrix.json" - @router.get("", response_model=MatrixResponse) -async def get_matrix(): - """Return the benchmark matrix.""" - if not MATRIX_PATH.exists(): +async def get_matrix(scenario: str = Query("cifar10_noniid")): + """Return the benchmark matrix for a given scenario.""" + from fl_core.research.scenarios import get_matrix_path_with_fallback + + matrix_path = get_matrix_path_with_fallback(scenario) + if matrix_path is None: raise HTTPException( 404, - "Benchmark matrix not found. Run 'arena generate' first.", + f"Benchmark matrix not found for scenario '{scenario}'. Run 'arena generate --scenario {scenario}' first.", ) - data = json.loads(MATRIX_PATH.read_text(encoding="utf-8")) + data = json.loads(matrix_path.read_text(encoding="utf-8")) return MatrixResponse( attacks=data.get("attacks", []), defenses=data.get("defenses", []), matrix=data.get("matrix", {}), config=data.get("config"), seeds=data.get("seeds"), + scenario_id=scenario, ) diff --git a/apps/backend/app/api/v1/router.py b/apps/backend/app/api/v1/router.py index dd8025c..2436f63 100644 --- a/apps/backend/app/api/v1/router.py +++ b/apps/backend/app/api/v1/router.py @@ -4,15 +4,19 @@ from .agent import router as agent_router from .bench import router as bench_router +from .dashboard import router as dashboard_router from .jobs import router as jobs_router from .leaderboard import router as leaderboard_router from .matrix import router as matrix_router +from .scenarios import router as scenarios_router from .submissions import router as submissions_router api_router = APIRouter(prefix="/api/v1") +api_router.include_router(dashboard_router) api_router.include_router(submissions_router) api_router.include_router(leaderboard_router) api_router.include_router(matrix_router) +api_router.include_router(scenarios_router) api_router.include_router(jobs_router) api_router.include_router(agent_router) api_router.include_router(bench_router) diff --git a/apps/backend/app/api/v1/scenarios.py b/apps/backend/app/api/v1/scenarios.py new file mode 100644 index 0000000..91d9185 --- /dev/null +++ b/apps/backend/app/api/v1/scenarios.py @@ -0,0 +1,25 @@ +"""Scenarios route — lists available FL evaluation scenarios.""" + +from __future__ import annotations + +from fastapi import APIRouter + +from ...schemas import ScenarioInfo + +router = APIRouter(prefix="/scenarios", tags=["scenarios"]) + + +@router.get("", response_model=list[ScenarioInfo]) +async def get_scenarios(): + from fl_core.research.scenarios import has_matrix, list_scenarios + + return [ + ScenarioInfo( + id=s.id, + name=s.name, + description=s.description, + has_matrix=has_matrix(s.id), + is_default=s.is_default, + ) + for s in list_scenarios() + ] diff --git a/apps/backend/app/api/v1/submissions.py b/apps/backend/app/api/v1/submissions.py index 216a3e3..0368bd1 100644 --- a/apps/backend/app/api/v1/submissions.py +++ b/apps/backend/app/api/v1/submissions.py @@ -2,22 +2,30 @@ from __future__ import annotations -from fastapi import APIRouter, Depends, HTTPException +import json +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import Response from sqlalchemy.ext.asyncio import AsyncSession from sqlmodel import select +from ...config import settings from ...db import get_session from ...models import EvaluationJob, Submission -from ...schemas import SubmissionCreate, SubmissionDetail, SubmissionResponse +from ...schemas import AnalysisResponse, SubmissionCreate, SubmissionDetail, SubmissionResponse, VersionInfo from ...services.evaluation import run_evaluation from ...services.report import generate_markdown_report from ...services.submission import ( + compute_versioned_name, remove_submission_files, + rewrite_method_name_in_code, validate_code, write_submission_files, ) +PROJECT_ROOT = Path(__file__).resolve().parents[5] + router = APIRouter(prefix="/submissions", tags=["submissions"]) @@ -35,18 +43,41 @@ async def create_submission( if not result["ok"]: raise HTTPException(422, result["error"]) - method_name: str = result["method_name"] + canonical_name: str = result["method_name"] class_name: str = result["class_name"] - # Check for duplicate method_name + # Check for existing submissions with the same method_group existing = ( - (await session.execute(select(Submission).where(Submission.method_name == method_name))).scalars().first() + ( + await session.execute( + select(Submission) + .where(Submission.method_group == canonical_name) + .order_by(Submission.version.desc()) + ) + ) + .scalars() + .all() ) + + if existing and not body.update_existing: + latest = existing[0] + raise HTTPException( + 409, + f"Method '{canonical_name}' already exists (v{latest.version or 1}, id={latest.id}). " + "Set update_existing=true to submit as a new version.", + ) + if existing: - raise HTTPException(409, f"Method '{method_name}' already exists (id={existing.id})") + next_version = (existing[0].version or 1) + 1 + method_name = compute_versioned_name(canonical_name, next_version) + code = rewrite_method_name_in_code(body.code, canonical_name, method_name) + else: + next_version = 1 + method_name = canonical_name + code = body.code # Write files to submissions/ - write_submission_files(method_name, class_name, body.role, body.code) + write_submission_files(method_name, class_name, body.role, code) # Create DB records submission = Submission( @@ -55,8 +86,10 @@ async def create_submission( display_name=body.display_name, author=body.author, description=body.description, - code=body.code, + code=code, status="evaluating", + method_group=canonical_name, + version=next_version, ) session.add(submission) await session.commit() @@ -68,7 +101,7 @@ async def create_submission( await session.refresh(job) # Launch background evaluation - run_evaluation(job.id, submission.id, method_name, body.role) + run_evaluation(job.id, submission.id, method_name, body.role, num_seeds=body.num_seeds) return SubmissionDetail( **submission.model_dump(), @@ -155,6 +188,107 @@ async def get_submission_report( ) +@router.get("/{submission_id}/analysis", response_model=AnalysisResponse) +async def get_submission_analysis( + submission_id: int, + regenerate: bool = Query(False), + session: AsyncSession = Depends(get_session), +): + """Get or generate LLM analysis for a completed submission.""" + if not settings.openai_api_key: + raise HTTPException(503, "LLM API key not configured") + + submission = await session.get(Submission, submission_id) + if not submission: + raise HTTPException(404, "Submission not found") + if submission.status != "completed": + raise HTTPException(400, "Submission evaluation not completed yet") + + stmt = ( + select(EvaluationJob) + .where(EvaluationJob.submission_id == submission_id) + .order_by(EvaluationJob.created_at.desc()) + ) + job = (await session.execute(stmt)).scalars().first() + if not job or not job.results_json: + raise HTTPException(400, "No evaluation results available") + + if job.analysis_text and not regenerate: + return AnalysisResponse(analysis=job.analysis_text, cached=True) + + from fl_core.research.scenarios import get_matrix_path_with_fallback + + matrix_path = get_matrix_path_with_fallback("cifar10_noniid") + if matrix_path is None: + raise HTTPException(404, "Benchmark matrix not found") + matrix_data = json.loads(matrix_path.read_text(encoding="utf-8")) + + results = json.loads(job.results_json) + + from ...services.analysis import generate_analysis + + analysis_text = await generate_analysis( + submission=submission.model_dump(), + results=results, + matrix_data=matrix_data, + ) + + job.analysis_text = analysis_text + session.add(job) + await session.commit() + + return AnalysisResponse(analysis=analysis_text, cached=False) + + +@router.get("/{submission_id}/versions", response_model=list[VersionInfo]) +async def get_submission_versions( + submission_id: int, + session: AsyncSession = Depends(get_session), +): + """Get all versions of the same method group.""" + submission = await session.get(Submission, submission_id) + if not submission: + raise HTTPException(404, "Submission not found") + if not submission.method_group: + return [] + + siblings = ( + await session.execute( + select(Submission) + .where(Submission.method_group == submission.method_group) + .order_by(Submission.version.desc()) + ) + ).scalars().all() + + result = [] + for sub in siblings: + avg_acc = None + job = ( + await session.execute( + select(EvaluationJob) + .where(EvaluationJob.submission_id == sub.id) + .order_by(EvaluationJob.created_at.desc()) + ) + ).scalars().first() + if job and job.results_json: + data = json.loads(job.results_json) + if "scenarios" in data: + data = data["scenarios"].get("cifar10_noniid", data) + accs = [v["avg_final_accuracy"] for v in data.values() if isinstance(v, dict) and v.get("avg_final_accuracy") is not None] + if accs: + avg_acc = sum(accs) / len(accs) + result.append(VersionInfo( + id=sub.id, + version=sub.version or 1, + method_name=sub.method_name, + display_name=sub.display_name, + status=sub.status, + avg_accuracy=avg_acc, + created_at=sub.created_at, + )) + return result + + @router.delete("/{submission_id}", status_code=204) async def delete_submission( submission_id: int, diff --git a/apps/backend/app/main.py b/apps/backend/app/main.py index 0c6a195..51e6fa0 100644 --- a/apps/backend/app/main.py +++ b/apps/backend/app/main.py @@ -54,9 +54,34 @@ async def _recover_stale_jobs() -> None: logger.info("Recovered %d stale jobs on startup", total_stale) +async def _migrate_schema() -> None: + """Add columns introduced after the initial schema.""" + from .db import engine + + from sqlalchemy import text + + async with engine.begin() as conn: + for stmt in [ + "ALTER TABLE evaluation_jobs ADD COLUMN analysis_text TEXT", + "ALTER TABLE evaluation_jobs ADD COLUMN total_scenarios INTEGER DEFAULT 0", + "ALTER TABLE evaluation_jobs ADD COLUMN completed_scenarios INTEGER DEFAULT 0", + "ALTER TABLE evaluation_jobs ADD COLUMN current_scenario TEXT", + "ALTER TABLE submissions ADD COLUMN method_group TEXT", + "ALTER TABLE submissions ADD COLUMN version INTEGER", + ]: + try: + await conn.execute(text(stmt)) + except Exception: + pass + await conn.execute( + text("UPDATE submissions SET method_group = method_name, version = 1 WHERE method_group IS NULL") + ) + + @asynccontextmanager async def lifespan(app: FastAPI): await init_db() + await _migrate_schema() init_queue(max_workers=1) await _recover_stale_jobs() yield @@ -65,14 +90,14 @@ async def lifespan(app: FastAPI): app = FastAPI( title="FedArena API", - version="0.2.0", + version="0.3.0", description="Attack/Defense evaluation arena for Federated Learning", lifespan=lifespan, ) app.add_middleware( CORSMiddleware, - allow_origins=["http://localhost:5173"], + allow_origins=["http://localhost:5173", "http://localhost:5174"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], diff --git a/apps/backend/app/models.py b/apps/backend/app/models.py index d4a6e97..d8052e7 100644 --- a/apps/backend/app/models.py +++ b/apps/backend/app/models.py @@ -20,6 +20,8 @@ class Submission(SQLModel, table=True): description: str | None = None code: str = Field(sa_column=Column(Text)) status: str = Field(default="pending") # pending | evaluating | completed | failed + method_group: str | None = Field(default=None, index=True) + version: int | None = None created_at: datetime = Field(default_factory=_utcnow) updated_at: datetime = Field(default_factory=_utcnow) @@ -34,7 +36,11 @@ class EvaluationJob(SQLModel, table=True): total_opponents: int = 0 completed_opponents: int = 0 current_opponent: str | None = None + total_scenarios: int = 0 + completed_scenarios: int = 0 + current_scenario: str | None = None results_json: str | None = None # JSON string of full eval results + analysis_text: str | None = Field(default=None, sa_column=Column(Text)) error: str | None = None started_at: datetime | None = None completed_at: datetime | None = None diff --git a/apps/backend/app/schemas.py b/apps/backend/app/schemas.py index 8a20a4a..5caf95b 100644 --- a/apps/backend/app/schemas.py +++ b/apps/backend/app/schemas.py @@ -13,6 +13,8 @@ class SubmissionCreate(BaseModel): display_name: str author: str | None = None description: str | None = None + num_seeds: int | None = None # None = use matrix default; 1 = quick; 3 = full + update_existing: bool = False class SubmissionResponse(BaseModel): @@ -23,6 +25,8 @@ class SubmissionResponse(BaseModel): author: str | None description: str | None status: str + method_group: str | None = None + version: int | None = None created_at: datetime updated_at: datetime @@ -67,14 +71,49 @@ class LeaderboardEntry(BaseModel): avg_accuracy: float opponent_scores: dict # {opponent_name: accuracy} submitted_at: datetime + avg_accuracy_drop: float | None = None + worst_case_accuracy: float | None = None + avg_convergence_speed: float | None = None + avg_stability: float | None = None + version: int | None = None + has_older_versions: bool = False + + +class VersionInfo(BaseModel): + id: int + version: int + method_name: str + display_name: str + status: str + avg_accuracy: float | None = None + created_at: datetime + + model_config = {"from_attributes": True} # ── Matrix ─────────────────────────────────────────────────── +class AnalysisResponse(BaseModel): + analysis: str + cached: bool + + class MatrixResponse(BaseModel): attacks: list[str] defenses: list[str] matrix: dict # {attack_key: {defense_key: {avg_final_accuracy, ...}}} config: str | None = None seeds: list[int] | None = None + scenario_id: str | None = None + + +# ── Scenarios ──────────────────────────────────────────────── + + +class ScenarioInfo(BaseModel): + id: str + name: str + description: str + has_matrix: bool + is_default: bool = False diff --git a/apps/backend/app/services/analysis.py b/apps/backend/app/services/analysis.py new file mode 100644 index 0000000..7e167fd --- /dev/null +++ b/apps/backend/app/services/analysis.py @@ -0,0 +1,148 @@ +"""LLM-generated analytical reports for arena submissions.""" + +from __future__ import annotations + +import logging +from typing import Any + +from openai import AsyncOpenAI + +from ..config import settings + +logger = logging.getLogger("fedarena.analysis") + +ANALYSIS_SYSTEM_PROMPT = """\ +You are FedArena's analysis assistant. Given a submission's evaluation results \ +against benchmark opponents, produce a structured analysis report in Markdown. + +## Output Structure + +1. **Performance Summary** — Overall metrics, how it compares to baselines +2. **Strengths** — Which opponents this method performs best against, with quantitative evidence +3. **Weaknesses** — Which opponents are most problematic, with quantitative evidence +4. **Comparison with Baselines** — How this method ranks among existing baseline methods in the matrix +5. **Key Observations** — Notable patterns in convergence, stability, or accuracy trajectory +6. **Recommendations** — Concrete suggestions for improving the method + +## Rules +- Use multi-metric data when available: accuracy_drop, convergence_speed, stability, max_accuracy, runtime +- Be specific — cite actual accuracy values and metric numbers +- For attacks: lower opponent accuracy = stronger attack +- For defenses: higher accuracy under attack = stronger defense +- Keep the report concise but insightful (400-700 words) +- Use Markdown formatting: headers (##), bold, bullet points +- Write in English +""" + + +def _get_client() -> AsyncOpenAI: + return AsyncOpenAI( + api_key=settings.openai_api_key, + base_url=settings.openai_api_base, + ) + + +def _build_analysis_context( + submission: dict[str, Any], + results: dict[str, Any], + matrix_data: dict[str, Any], +) -> str: + parts: list[str] = [] + + parts.append(f"## Submission: {submission.get('display_name', submission.get('method_name'))}") + parts.append(f"- Role: {submission['role']}") + parts.append(f"- Method: `{submission['method_name']}`") + if submission.get("description"): + parts.append(f"- Description: {submission['description']}") + parts.append("") + + summary = results.get("__summary__", {}) + if summary: + parts.append("## Overall Metrics") + for key, label in [ + ("avg_accuracy", "Avg Accuracy"), + ("avg_accuracy_drop", "Avg Accuracy Drop"), + ("worst_case_accuracy", "Worst-Case Accuracy"), + ("best_case_accuracy", "Best-Case Accuracy"), + ("avg_convergence_speed", "Avg Convergence Speed (rounds)"), + ("avg_stability", "Avg Stability (std)"), + ("avg_runtime_seconds", "Avg Runtime (seconds)"), + ]: + if summary.get(key) is not None: + val = summary[key] + parts.append(f"- {label}: {val:.4f}" if isinstance(val, float) else f"- {label}: {val}") + parts.append("") + + parts.append("## Per-Opponent Results") + parts.append("| Opponent | Accuracy | Acc. Drop | Baseline | Convergence | Stability |") + parts.append("|----------|----------|-----------|----------|-------------|-----------|") + for opp, res in results.items(): + if opp == "__summary__": + continue + opp_label = opp.replace("__none__", "none").replace("baseline_", "") + acc = f"{res['avg_final_accuracy']:.4f}" if res.get("avg_final_accuracy") is not None else "FAIL" + drop = f"{res['accuracy_drop']:.4f}" if res.get("accuracy_drop") is not None else "-" + base = f"{res['baseline_accuracy']:.4f}" if res.get("baseline_accuracy") is not None else "-" + conv = f"{res['avg_convergence_speed']:.0f}" if res.get("avg_convergence_speed") is not None else "-" + stab = f"{res['avg_stability']:.4f}" if res.get("avg_stability") is not None else "-" + parts.append(f"| {opp_label} | {acc} | {drop} | {base} | {conv} | {stab} |") + parts.append("") + + matrix = matrix_data.get("matrix", {}) + role = submission["role"] + if matrix: + parts.append("## Baseline Reference Matrix (for comparison)") + if role == "attack": + parts.append("Baseline attack accuracies (lower = stronger attack):") + for atk_name, defenses in matrix.items(): + if atk_name == "__none__": + continue + accs = [v.get("avg_final_accuracy") for v in defenses.values() if v.get("avg_final_accuracy") is not None] + if accs: + avg = sum(accs) / len(accs) + parts.append(f"- {atk_name}: avg={avg:.4f}") + else: + parts.append("Baseline defense accuracies (higher = stronger defense):") + all_defenses: dict[str, list[float]] = {} + for _atk, defenses in matrix.items(): + for dfn, vals in defenses.items(): + if dfn == "__none__": + continue + acc = vals.get("avg_final_accuracy") + if acc is not None: + all_defenses.setdefault(dfn, []).append(acc) + for dfn, accs in all_defenses.items(): + avg = sum(accs) / len(accs) + parts.append(f"- {dfn}: avg={avg:.4f}") + parts.append("") + + parts.append("Please analyze this submission's performance and provide insights.") + return "\n".join(parts) + + +def _extract_default_scenario_results(results: dict[str, Any]) -> dict[str, Any]: + """Extract results for analysis — use default scenario from nested format, or flat results.""" + if "scenarios" in results: + return results["scenarios"].get("cifar10_noniid", next(iter(results["scenarios"].values()), {})) + return results + + +async def generate_analysis( + submission: dict[str, Any], + results: dict[str, Any], + matrix_data: dict[str, Any], +) -> str: + client = _get_client() + scenario_results = _extract_default_scenario_results(results) + context = _build_analysis_context(submission, scenario_results, matrix_data) + + response = await client.chat.completions.create( + model=settings.default_llm_model, + max_tokens=settings.max_tokens, + messages=[ + {"role": "system", "content": ANALYSIS_SYSTEM_PROMPT}, + {"role": "user", "content": context}, + ], + ) + + return response.choices[0].message.content or "" diff --git a/apps/backend/app/services/evaluation.py b/apps/backend/app/services/evaluation.py index 7da72c6..9604d88 100644 --- a/apps/backend/app/services/evaluation.py +++ b/apps/backend/app/services/evaluation.py @@ -8,6 +8,7 @@ import json import logging +import statistics import sys import threading from datetime import UTC, datetime @@ -58,7 +59,13 @@ def _update_submission(submission_id: int, **fields) -> None: session.commit() -def run_evaluation(job_id: int, submission_id: int, method_name: str, role: str) -> None: +def run_evaluation( + job_id: int, + submission_id: int, + method_name: str, + role: str, + num_seeds: int | None = None, +) -> None: """Submit evaluation to the GPU task queue.""" from .task_queue import get_queue @@ -77,6 +84,7 @@ def on_cancel() -> None: submission_id, method_name, role, + num_seeds, on_cancel=on_cancel, ) @@ -86,6 +94,7 @@ def _evaluation_worker( submission_id: int, method_name: str, role: str, + num_seeds: int | None, cancel_event: threading.Event, ) -> None: def now() -> datetime: @@ -116,98 +125,68 @@ def now() -> datetime: else: reg.get_defense(method_name) - matrix_path = PROJECT_ROOT / "results" / "arena" / "benchmark_matrix.json" - if not matrix_path.exists(): - raise FileNotFoundError("Benchmark matrix not found. Run 'arena generate' first.") + from fl_core.research.scenarios import get_config_path, get_matrix_path_with_fallback, list_scenarios - matrix_data = json.loads(matrix_path.read_text(encoding="utf-8")) + scenarios_with_matrix = [] + for s in list_scenarios(): + mp = get_matrix_path_with_fallback(s.id) + if mp is not None: + scenarios_with_matrix.append((s.id, mp, get_config_path(s.id))) - if role == "attack": - opponents = matrix_data["defenses"] - else: - opponents = matrix_data["attacks"] + if not scenarios_with_matrix: + raise FileNotFoundError("No benchmark matrices found. Run 'arena generate' first.") - total = len(opponents) - _update_job(job_id, total_opponents=total) + total_scenarios = len(scenarios_with_matrix) + _update_job(job_id, total_scenarios=total_scenarios) - config_path = PROJECT_ROOT / "configs" / "research" / "bench_baseline.yaml" - config = _load_config(config_path) - output_dir = PROJECT_ROOT / "results" / "arena" - seeds = matrix_data.get("seeds", [0]) + all_scenario_results: dict[str, dict[str, Any]] = {} - results: dict[str, Any] = {} - NO_ATTACK = "__none__" - NO_DEFENSE = "__none__" - - for i, opp in enumerate(opponents): + for sc_idx, (scenario_id, matrix_path, config_path) in enumerate(scenarios_with_matrix): if cancel_event.is_set(): _update_job(job_id, status="cancelled", completed_at=now()) _update_submission(submission_id, status="failed", updated_at=now()) logger.info("Evaluation cancelled: %s", method_name) return - _update_job( - job_id, - completed_opponents=i, - current_opponent=opp, - progress=f"{i}/{total} opponents done", + _update_job(job_id, current_scenario=scenario_id, completed_scenarios=sc_idx) + + scenario_results = _run_scenario_opponents( + job_id=job_id, + method_name=method_name, + role=role, + matrix_path=matrix_path, + config_path=config_path, + scenario_id=scenario_id, + scenario_idx=sc_idx, + total_scenarios=total_scenarios, + num_seeds=num_seeds, + cancel_event=cancel_event, + now=now, ) - if role == "attack": - atk = method_name - dfn = opp if opp != NO_DEFENSE else None - else: - atk = opp if opp != NO_ATTACK else None - dfn = method_name - - seed_results = [] - for seed in seeds: - try: - from fl_core.research.runner import _run_single_seed - - metrics = _run_single_seed( - config=config, - experiment_name=f"{atk or NO_ATTACK}_vs_{dfn or NO_DEFENSE}", - seed=seed, - results_dir=output_dir / "runs", - attack_method=atk, - defense_method=dfn, - ) - acc = metrics.get("final_accuracy") - global_res = metrics.get("global_results", {}) - seed_results.append( - { - "seed": seed, - "final_accuracy": acc, - "rounds": global_res.get("rounds", []), - "accuracy_trajectory": global_res.get("global_accuracy", []), - "loss_trajectory": global_res.get("global_loss", []), - } - ) - except Exception as e: - logger.error("Seed %d failed: %s", seed, e) - seed_results.append({"seed": seed, "final_accuracy": None, "error": str(e)}) - - valid = [r["final_accuracy"] for r in seed_results if r["final_accuracy"] is not None] - results[opp] = { - "avg_final_accuracy": sum(valid) / len(valid) if valid else None, - "per_seed": seed_results, - } - - _update_job( - job_id, - completed_opponents=i + 1, - progress=f"{i + 1}/{total} opponents done", - results_json=json.dumps(results, ensure_ascii=False), - ) + if scenario_results is None: + _update_submission(submission_id, status="failed", updated_at=now()) + return + + all_scenario_results[scenario_id] = scenario_results + + if len(all_scenario_results) == 1 and "cifar10_noniid" in all_scenario_results: + final_results = all_scenario_results["cifar10_noniid"] + else: + final_results = {"scenarios": all_scenario_results} + global_summary = _compute_global_summary(all_scenario_results) + if global_summary: + final_results["__global_summary__"] = global_summary + output_dir = PROJECT_ROOT / "results" / "arena" _update_job( job_id, status="completed", - completed_opponents=total, current_opponent=None, - progress=f"{total}/{total} opponents done", - results_json=json.dumps(results, ensure_ascii=False), + current_scenario=None, + completed_scenarios=total_scenarios, + progress="completed", + results_json=json.dumps(final_results, ensure_ascii=False), completed_at=now(), ) _update_submission(submission_id, status="completed", updated_at=now()) @@ -215,11 +194,11 @@ def now() -> datetime: eval_path = output_dir / f"eval_{method_name}.json" eval_path.parent.mkdir(parents=True, exist_ok=True) eval_path.write_text( - json.dumps({"method": method_name, "role": role, "results": results}, indent=2, ensure_ascii=False), + json.dumps({"method": method_name, "role": role, "results": final_results}, indent=2, ensure_ascii=False), encoding="utf-8", ) - logger.info("Evaluation completed: %s", method_name) + logger.info("Evaluation completed: %s (%d scenarios)", method_name, total_scenarios) except Exception as e: logger.exception("Evaluation failed for %s", method_name) @@ -227,6 +206,178 @@ def now() -> datetime: _update_submission(submission_id, status="failed", updated_at=now()) +def _run_scenario_opponents( + *, + job_id: int, + method_name: str, + role: str, + matrix_path: Path, + config_path: Path, + scenario_id: str, + scenario_idx: int, + total_scenarios: int, + num_seeds: int | None = None, + cancel_event: threading.Event, + now, +) -> dict[str, Any] | None: + """Run all opponent matches for one scenario. Returns results dict or None if cancelled.""" + NO_ATTACK = "__none__" + NO_DEFENSE = "__none__" + + matrix_data = json.loads(matrix_path.read_text(encoding="utf-8")) + config = _load_config(config_path) + output_dir = PROJECT_ROOT / "results" / "arena" + seeds = matrix_data.get("seeds", [0]) + if num_seeds is not None and num_seeds > 0: + seeds = list(range(num_seeds)) + matrix = matrix_data.get("matrix", {}) + + opponents = matrix_data["defenses"] if role == "attack" else matrix_data["attacks"] + total = len(opponents) + _update_job(job_id, total_opponents=total, completed_opponents=0) + + sc_label = f"{scenario_id} ({scenario_idx + 1}/{total_scenarios})" if total_scenarios > 1 else "" + + results: dict[str, Any] = {} + + for i, opp in enumerate(opponents): + if cancel_event.is_set(): + _update_job(job_id, status="cancelled", completed_at=now()) + logger.info("Evaluation cancelled: %s", method_name) + return None + + progress = f"{sc_label} {i}/{total} opponents" if sc_label else f"{i}/{total} opponents done" + _update_job(job_id, completed_opponents=i, current_opponent=opp, progress=progress) + + if role == "attack": + atk = method_name + dfn = opp if opp != NO_DEFENSE else None + else: + atk = opp if opp != NO_ATTACK else None + dfn = method_name + + seed_results = [] + for seed in seeds: + try: + from fl_core.research.runner import _run_single_seed + + metrics = _run_single_seed( + config=config, + experiment_name=f"{atk or NO_ATTACK}_vs_{dfn or NO_DEFENSE}", + seed=seed, + results_dir=output_dir / "runs", + attack_method=atk, + defense_method=dfn, + ) + acc = metrics.get("final_accuracy") + global_res = metrics.get("global_results", {}) + seed_results.append( + { + "seed": seed, + "final_accuracy": acc, + "initial_accuracy": metrics.get("initial_accuracy"), + "max_accuracy": metrics.get("max_accuracy"), + "convergence_speed": metrics.get("convergence_speed"), + "stability": metrics.get("stability"), + "runtime_seconds": metrics.get("runtime_seconds"), + "rounds": global_res.get("rounds", []), + "accuracy_trajectory": global_res.get("global_accuracy", []), + "loss_trajectory": global_res.get("global_loss", []), + } + ) + except Exception as e: + logger.error("Seed %d failed: %s", seed, e) + seed_results.append({"seed": seed, "final_accuracy": None, "error": str(e)}) + + valid_acc = [r["final_accuracy"] for r in seed_results if r["final_accuracy"] is not None] + avg_acc = sum(valid_acc) / len(valid_acc) if valid_acc else None + + if role == "attack": + baseline_acc = matrix.get(NO_ATTACK, {}).get(opp, {}).get("avg_final_accuracy") + else: + baseline_acc = matrix.get(opp, {}).get(NO_DEFENSE, {}).get("avg_final_accuracy") + + accuracy_drop = None + if avg_acc is not None and baseline_acc is not None: + accuracy_drop = baseline_acc - avg_acc + + def _avg(key: str) -> float | None: + vals = [r[key] for r in seed_results if r.get(key) is not None] + return sum(vals) / len(vals) if vals else None + + results[opp] = { + "avg_final_accuracy": avg_acc, + "accuracy_drop": accuracy_drop, + "baseline_accuracy": baseline_acc, + "max_accuracy": _avg("max_accuracy"), + "avg_convergence_speed": _avg("convergence_speed"), + "avg_stability": _avg("stability"), + "avg_runtime_seconds": _avg("runtime_seconds"), + "std_final_accuracy": statistics.stdev(valid_acc) if len(valid_acc) > 1 else 0.0, + "per_seed": seed_results, + } + + progress = f"{sc_label} {i + 1}/{total} opponents" if sc_label else f"{i + 1}/{total} opponents done" + _update_job( + job_id, + completed_opponents=i + 1, + progress=progress, + results_json=json.dumps(results, ensure_ascii=False), + ) + + results["__summary__"] = _compute_scenario_summary(results) + return results + + +def _compute_scenario_summary(results: dict[str, Any]) -> dict[str, Any]: + all_accs = [r["avg_final_accuracy"] for r in results.values() if r.get("avg_final_accuracy") is not None] + all_drops = [r["accuracy_drop"] for r in results.values() if r.get("accuracy_drop") is not None] + all_conv = [r["avg_convergence_speed"] for r in results.values() if r.get("avg_convergence_speed") is not None] + all_stab = [r["avg_stability"] for r in results.values() if r.get("avg_stability") is not None] + all_rt = [r["avg_runtime_seconds"] for r in results.values() if r.get("avg_runtime_seconds") is not None] + + summary: dict[str, Any] = {} + if all_accs: + summary["avg_accuracy"] = sum(all_accs) / len(all_accs) + summary["worst_case_accuracy"] = min(all_accs) + summary["best_case_accuracy"] = max(all_accs) + if all_drops: + summary["avg_accuracy_drop"] = sum(all_drops) / len(all_drops) + if all_conv: + summary["avg_convergence_speed"] = sum(all_conv) / len(all_conv) + if all_stab: + summary["avg_stability"] = sum(all_stab) / len(all_stab) + if all_rt: + summary["avg_runtime_seconds"] = sum(all_rt) / len(all_rt) + return summary + + +def _compute_global_summary(all_scenario_results: dict[str, dict[str, Any]]) -> dict[str, Any]: + """Aggregate summaries across all scenarios.""" + summaries = [s.get("__summary__", {}) for s in all_scenario_results.values() if s.get("__summary__")] + if not summaries: + return {} + + def _avg_key(key: str) -> float | None: + vals = [s[key] for s in summaries if key in s and s[key] is not None] + return sum(vals) / len(vals) if vals else None + + result: dict[str, Any] = {} + for key in ("avg_accuracy", "avg_accuracy_drop", "avg_convergence_speed", "avg_stability", "avg_runtime_seconds"): + val = _avg_key(key) + if val is not None: + result[key] = val + + worst = [s["worst_case_accuracy"] for s in summaries if "worst_case_accuracy" in s] + if worst: + result["worst_case_accuracy"] = min(worst) + best = [s["best_case_accuracy"] for s in summaries if "best_case_accuracy" in s] + if best: + result["best_case_accuracy"] = max(best) + + return result + + def _load_config(path: Path) -> dict: import yaml diff --git a/apps/backend/app/services/report.py b/apps/backend/app/services/report.py index 513e5b8..36605a5 100644 --- a/apps/backend/app/services/report.py +++ b/apps/backend/app/services/report.py @@ -30,7 +30,13 @@ def generate_markdown_report( lines.append("") if results: - _append_results(lines, results, role) + if "scenarios" in results: + for sc_id, sc_results in results["scenarios"].items(): + lines.append(f"## Scenario: {sc_id}") + lines.append("") + _append_results(lines, sc_results, role) + else: + _append_results(lines, results, role) lines.append("## Code") lines.append("") @@ -50,45 +56,47 @@ def _append_results( results: dict[str, Any], role: str, ) -> None: - accs = [r["avg_final_accuracy"] for r in results.values() if r.get("avg_final_accuracy") is not None] + summary = results.get("__summary__", {}) + opponent_results = {k: v for k, v in results.items() if k != "__summary__"} + + accs = [r["avg_final_accuracy"] for r in opponent_results.values() if r.get("avg_final_accuracy") is not None] avg = sum(accs) / len(accs) if accs else None lines.append("## Summary") lines.append("") if avg is not None: lines.append(f"- **Overall Average Accuracy:** {avg:.4f}") - lines.append(f"- **Opponents Evaluated:** {len(results)}") + if summary.get("avg_accuracy_drop") is not None: + lines.append(f"- **Average Accuracy Drop:** {summary['avg_accuracy_drop']:.4f}") + if summary.get("worst_case_accuracy") is not None: + lines.append(f"- **Worst-Case Accuracy:** {summary['worst_case_accuracy']:.4f}") + if summary.get("avg_convergence_speed") is not None: + lines.append(f"- **Average Convergence Speed:** {summary['avg_convergence_speed']:.1f} rounds") + if summary.get("avg_stability") is not None: + lines.append(f"- **Average Stability (std):** {summary['avg_stability']:.4f}") + if summary.get("avg_runtime_seconds") is not None: + lines.append(f"- **Average Runtime:** {summary['avg_runtime_seconds']:.1f}s") + lines.append(f"- **Opponents Evaluated:** {len(opponent_results)}") interpretation = "lower = stronger attack" if role == "attack" else "higher = stronger defense" lines.append(f"- **Interpretation:** {interpretation}") lines.append("") - multi_seed = any(len(r.get("per_seed", [])) > 1 for r in results.values()) - lines.append("## Per-Opponent Results") lines.append("") + lines.append("| Opponent | Accuracy | Acc. Drop | Convergence | Stability |") + lines.append("|----------|----------|-----------|-------------|-----------|") - if multi_seed: - lines.append("| Opponent | Avg Accuracy | Per Seed |") - lines.append("|----------|-------------|----------|") - else: - lines.append("| Opponent | Accuracy |") - lines.append("|----------|----------|") - - for opp, res in results.items(): + for opp, res in opponent_results.items(): opp_label = opp.replace("__none__", "none").replace("baseline_", "") acc_str = f"{res['avg_final_accuracy']:.4f}" if res.get("avg_final_accuracy") is not None else "FAIL" - if multi_seed: - seeds_str = ", ".join( - f"{s['final_accuracy']:.4f}" if s.get("final_accuracy") is not None else "fail" - for s in res.get("per_seed", []) - ) - lines.append(f"| {opp_label} | {acc_str} | {seeds_str} |") - else: - lines.append(f"| {opp_label} | {acc_str} |") + drop_str = f"{res['accuracy_drop']:.4f}" if res.get("accuracy_drop") is not None else "-" + conv_str = f"{res['avg_convergence_speed']:.0f}" if res.get("avg_convergence_speed") is not None else "-" + stab_str = f"{res['avg_stability']:.4f}" if res.get("avg_stability") is not None else "-" + lines.append(f"| {opp_label} | {acc_str} | {drop_str} | {conv_str} | {stab_str} |") lines.append("") - _append_trajectories(lines, results) + _append_trajectories(lines, opponent_results) def _append_trajectories( diff --git a/apps/backend/app/services/submission.py b/apps/backend/app/services/submission.py index ec0ed1f..92d7da9 100644 --- a/apps/backend/app/services/submission.py +++ b/apps/backend/app/services/submission.py @@ -106,6 +106,22 @@ def write_submission_files(method_name: str, class_name: str, role: str, code: s return sub_dir +def compute_versioned_name(group: str, version: int) -> str: + """Return the actual method_name for a given group and version.""" + if version <= 1: + return group + return f"{group}_v{version}" + + +def rewrite_method_name_in_code(code: str, old_name: str, new_name: str) -> str: + """Replace method_name assignment value in strategy code.""" + return re.sub( + r'(method_name\s*=\s*["\'])' + re.escape(old_name) + r'(["\'])', + r"\g<1>" + new_name + r"\2", + code, + ) + + def remove_submission_files(method_name: str, role: str) -> None: """Remove submission directory.""" import shutil diff --git a/apps/backend/tests/test_dashboard.py b/apps/backend/tests/test_dashboard.py new file mode 100644 index 0000000..8765110 --- /dev/null +++ b/apps/backend/tests/test_dashboard.py @@ -0,0 +1,87 @@ +"""Tests for the dashboard aggregation endpoint.""" + +from __future__ import annotations + +import json + +import pytest +from httpx import AsyncClient + +from .conftest import VALID_ATTACK_CODE, VALID_DEFENSE_CODE, test_session + + +@pytest.mark.asyncio +async def test_dashboard_empty(client: AsyncClient): + resp = await client.get("/api/v1/dashboard") + assert resp.status_code == 200 + data = resp.json() + assert data["total_submissions"] == 0 + assert data["completed_evaluations"] == 0 + assert data["failed_evaluations"] == 0 + assert data["active_jobs"] == 0 + assert data["top_attack"] is None + assert data["top_defense"] is None + assert data["recent_submissions"] == [] + + +@pytest.mark.asyncio +async def test_dashboard_counts_submissions(client: AsyncClient): + await client.post( + "/api/v1/submissions", + json={"code": VALID_ATTACK_CODE, "role": "attack", "display_name": "Atk1"}, + ) + await client.post( + "/api/v1/submissions", + json={"code": VALID_DEFENSE_CODE, "role": "defense", "display_name": "Def1"}, + ) + resp = await client.get("/api/v1/dashboard") + data = resp.json() + assert data["total_submissions"] == 2 + assert len(data["recent_submissions"]) == 2 + + +@pytest.mark.asyncio +async def test_dashboard_recent_order(client: AsyncClient): + await client.post( + "/api/v1/submissions", + json={"code": VALID_ATTACK_CODE, "role": "attack", "display_name": "Atk"}, + ) + await client.post( + "/api/v1/submissions", + json={"code": VALID_DEFENSE_CODE, "role": "defense", "display_name": "Def"}, + ) + resp = await client.get("/api/v1/dashboard") + data = resp.json() + names = [s["display_name"] for s in data["recent_submissions"]] + assert names == ["Def", "Atk"] + + +@pytest.mark.asyncio +async def test_dashboard_top_methods(client: AsyncClient): + from apps.backend.app.models import EvaluationJob, Submission + + async with test_session() as session: + sub = Submission( + method_name="arena_attack_test", + role="attack", + display_name="Good Attack", + code=VALID_ATTACK_CODE, + status="completed", + ) + session.add(sub) + await session.flush() + + job = EvaluationJob( + submission_id=sub.id, + status="completed", + results_json=json.dumps({"opp1": {"avg_final_accuracy": 0.15}}), + ) + session.add(job) + await session.commit() + + resp = await client.get("/api/v1/dashboard") + data = resp.json() + assert data["completed_evaluations"] == 1 + assert data["top_attack"] is not None + assert data["top_attack"]["display_name"] == "Good Attack" + assert data["top_attack"]["avg_accuracy"] == pytest.approx(0.15) diff --git a/apps/backend/tests/test_submissions.py b/apps/backend/tests/test_submissions.py index b398175..7315044 100644 --- a/apps/backend/tests/test_submissions.py +++ b/apps/backend/tests/test_submissions.py @@ -470,9 +470,8 @@ def test_multi_seed_report(self): }, } md = generate_markdown_report(sub, results) - assert "Per Seed" in md - assert "0.7800" in md - assert "0.8200" in md + assert "Per-Opponent Results" in md + assert "0.8000" in md def test_trajectory_table(self): sub = { diff --git a/apps/frontend/src/App.tsx b/apps/frontend/src/App.tsx index e286d1f..5fb12cc 100644 --- a/apps/frontend/src/App.tsx +++ b/apps/frontend/src/App.tsx @@ -1,65 +1,156 @@ +import { useState } from "react"; import { BrowserRouter, Link, NavLink, Route, Routes } from "react-router-dom"; -import { Trophy, FlaskConical, Swords } from "lucide-react"; +import { + LayoutDashboard, + Trophy, + FlaskConical, + Swords, + Activity, + GitCompareArrows, + Menu, + X, +} from "lucide-react"; +import Dashboard from "./pages/Dashboard"; import Leaderboard from "./pages/Leaderboard"; import Bench from "./pages/Bench"; import Arena from "./pages/Arena"; +import Compare from "./pages/Compare"; import Detail from "./pages/Detail"; +import BenchDetail from "./pages/BenchDetail"; +import Jobs from "./pages/Jobs"; import ThemeToggle from "./components/ThemeToggle"; import SettingsDialog from "./components/SettingsDialog"; +import { AgentProvider } from "./context/AgentContext"; +import { BenchProvider } from "./context/BenchContext"; + +const NAV_ITEMS: { to: string; icon: React.ReactNode; label: string; section: string }[] = [ + { to: "/", icon: , label: "Dashboard", section: "Overview" }, + { to: "/arena", icon: , label: "Arena", section: "Submit" }, + { to: "/bench", icon: , label: "Bench", section: "Submit" }, + { to: "/leaderboard", icon: , label: "Leaderboard", section: "Results" }, + { to: "/compare", icon: , label: "Compare", section: "Results" }, + { to: "/jobs", icon: , label: "Jobs", section: "Results" }, +]; function App() { + const [sidebarOpen, setSidebarOpen] = useState(false); + return ( -

- {/* Nav */} - + - {/* Content */} -
- - } /> - } /> - } /> - } /> - -
+
+ + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + +
+
+ + ); } -function NavItem({ to, icon, label }: { to: string; icon: React.ReactNode; label: string }) { +function SidebarNav() { + let lastSection = ""; return ( - - `flex items-center gap-1.5 px-3 py-1.5 rounded-md text-sm transition-colors ${ - isActive - ? "bg-blue-600/20 text-blue-400" - : "text-slate-400 hover:text-slate-200 hover:bg-slate-800" - }` - } - > - {icon} - {label} - +
+ {NAV_ITEMS.map((item) => { + const showSection = item.section !== lastSection; + lastSection = item.section; + return ( +
+ {showSection && ( +

+ {item.section} +

+ )} + + `flex items-center gap-2 px-3 py-2 rounded-md text-sm transition-colors ${ + isActive + ? "bg-blue-600/20 text-blue-400" + : "text-slate-400 hover:text-slate-200 hover:bg-slate-800" + }` + } + > + {item.icon} + {item.label} + +
+ ); + })} +
); } diff --git a/apps/frontend/src/api/client.ts b/apps/frontend/src/api/client.ts index 576bdd3..cd01221 100644 --- a/apps/frontend/src/api/client.ts +++ b/apps/frontend/src/api/client.ts @@ -23,6 +23,8 @@ export interface Submission { author: string | null; description: string | null; status: string; + method_group: string | null; + version: number | null; created_at: string; updated_at: string; } @@ -30,7 +32,7 @@ export interface Submission { export interface SubmissionDetail extends Submission { code: string; job_id: number | null; - results: Record | null; + results: Record | null; error: string | null; } @@ -41,13 +43,34 @@ export interface SeedResult { rounds?: number[]; accuracy_trajectory?: number[]; loss_trajectory?: number[]; + max_accuracy?: number | null; + convergence_speed?: number | null; + stability?: number | null; + runtime_seconds?: number | null; } export interface OpponentResult { avg_final_accuracy: number | null; + accuracy_drop?: number | null; + baseline_accuracy?: number | null; + max_accuracy?: number | null; + avg_convergence_speed?: number | null; + avg_stability?: number | null; + avg_runtime_seconds?: number | null; + std_final_accuracy?: number | null; per_seed: SeedResult[]; } +export interface ResultsSummary { + avg_accuracy?: number; + avg_accuracy_drop?: number; + worst_case_accuracy?: number; + best_case_accuracy?: number; + avg_convergence_speed?: number; + avg_stability?: number; + avg_runtime_seconds?: number; +} + export interface LeaderboardEntry { rank: number; submission_id: number; @@ -58,6 +81,27 @@ export interface LeaderboardEntry { avg_accuracy: number; opponent_scores: Record; submitted_at: string; + avg_accuracy_drop?: number | null; + worst_case_accuracy?: number | null; + avg_convergence_speed?: number | null; + avg_stability?: number | null; + version?: number | null; + has_older_versions?: boolean; +} + +export interface VersionInfo { + id: number; + version: number; + method_name: string; + display_name: string; + status: string; + avg_accuracy: number | null; + created_at: string; +} + +export interface AnalysisResponse { + analysis: string; + cached: boolean; } export interface JobProgress { @@ -73,12 +117,27 @@ export interface JobProgress { completed_at: string | null; } +export interface MatrixCellData { + avg_final_accuracy: number | null; + per_seed?: { seed: number; final_accuracy: number | null; error?: string | null }[]; + seeds_succeeded?: number; +} + export interface MatrixData { attacks: string[]; defenses: string[]; - matrix: Record>; + matrix: Record>; config: string | null; seeds: number[] | null; + scenario_id: string | null; +} + +export interface ScenarioInfo { + id: string; + name: string; + description: string; + has_matrix: boolean; + is_default: boolean; } export interface AgentConfig { @@ -133,6 +192,7 @@ export interface BenchJob { error: string | null; started_at: string | null; completed_at: string | null; + created_at: string | null; } export interface BenchResult { @@ -144,9 +204,51 @@ export interface BenchResult { error: string | null; } +export interface TopMethod { + submission_id: number; + display_name: string; + method_name: string; + author: string | null; + avg_accuracy: number; +} + +export interface DashboardData { + total_submissions: number; + completed_evaluations: number; + failed_evaluations: number; + active_jobs: number; + queue_pending: number; + top_attack: TopMethod | null; + top_defense: TopMethod | null; + recent_submissions: { + id: number; + display_name: string; + method_name: string; + role: string; + status: string; + created_at: string; + }[]; +} + +export interface JobListItem { + id: number; + job_type: "evaluation" | "bench"; + status: string; + label: string; + submission_id: number | null; + progress: string | null; + error: string | null; + started_at: string | null; + completed_at: string | null; + created_at: string; +} + // ── API functions ────────────────────────────────────────── export const api = { + // Dashboard + getDashboard: () => request("/dashboard"), + // Submissions createSubmission: (data: { code: string; @@ -154,6 +256,8 @@ export const api = { display_name: string; author?: string; description?: string; + num_seeds?: number; + update_existing?: boolean; }) => request("/submissions", { method: "POST", body: JSON.stringify(data), @@ -167,17 +271,34 @@ export const api = { getSubmissionReportUrl: (id: number) => `${BASE}/submissions/${id}/report`, + getVersions: (id: number) => + request(`/submissions/${id}/versions`), + deleteSubmission: (id: number) => request(`/submissions/${id}`, { method: "DELETE" }), + // Analysis + getAnalysis: (submissionId: number, regenerate?: boolean) => + request( + `/submissions/${submissionId}/analysis${regenerate ? "?regenerate=true" : ""}` + ), + + // Scenarios + getScenarios: () => request("/scenarios"), + // Leaderboard - getLeaderboard: (role: string) => - request(`/leaderboard?role=${role}`), + getLeaderboard: (role: string, sortBy?: string, scenario?: string, showAllVersions?: boolean) => + request( + `/leaderboard?role=${role}${sortBy ? `&sort_by=${sortBy}` : ""}${scenario ? `&scenario=${scenario}` : ""}${showAllVersions ? "&show_all_versions=true" : ""}` + ), // Matrix - getMatrix: () => request("/matrix"), + getMatrix: (scenario?: string) => + request(`/matrix${scenario ? `?scenario=${scenario}` : ""}`), // Jobs + listJobs: () => request("/jobs"), + getJobProgress: (jobId: number) => request(`/jobs/${jobId}/progress`), diff --git a/apps/frontend/src/components/IntensitySelector.tsx b/apps/frontend/src/components/IntensitySelector.tsx new file mode 100644 index 0000000..f083054 --- /dev/null +++ b/apps/frontend/src/components/IntensitySelector.tsx @@ -0,0 +1,39 @@ +const OPTIONS = [ + { value: 1, label: "Quick", desc: "1 seed, fast results" }, + { value: 0, label: "Standard", desc: "Matrix default seeds" }, + { value: 3, label: "Full", desc: "3 seeds, more reliable" }, +]; + +interface Props { + value: number; + onChange: (v: number) => void; + disabled?: boolean; +} + +export default function IntensitySelector({ value, onChange, disabled }: Props) { + return ( +
+ +
+ {OPTIONS.map((opt) => ( + + ))} +
+

+ {OPTIONS.find((o) => o.value === value)?.desc} +

+
+ ); +} diff --git a/apps/frontend/src/components/ScenarioSelector.tsx b/apps/frontend/src/components/ScenarioSelector.tsx new file mode 100644 index 0000000..9dcd395 --- /dev/null +++ b/apps/frontend/src/components/ScenarioSelector.tsx @@ -0,0 +1,32 @@ +import { useEffect, useState } from "react"; +import { api } from "../api/client"; +import type { ScenarioInfo } from "../api/client"; + +interface Props { + value: string; + onChange: (id: string) => void; +} + +export default function ScenarioSelector({ value, onChange }: Props) { + const [scenarios, setScenarios] = useState([]); + + useEffect(() => { + api.getScenarios().then(setScenarios).catch(() => {}); + }, []); + + if (scenarios.length === 0) return null; + + return ( + + ); +} diff --git a/apps/frontend/src/context/AgentContext.tsx b/apps/frontend/src/context/AgentContext.tsx new file mode 100644 index 0000000..c5ccea1 --- /dev/null +++ b/apps/frontend/src/context/AgentContext.tsx @@ -0,0 +1,93 @@ +import { createContext, useContext, useState, useCallback } from "react"; +import type { ReactNode } from "react"; +import { api } from "../api/client"; + +type Phase = "idle" | "generating" | "review" | "submitting"; + +interface AgentState { + prompt: string; + phase: Phase; + error: string | null; + code: string; + role: "attack" | "defense"; + displayName: string; + description: string; + numSeeds: number; +} + +interface AgentContextValue extends AgentState { + setPrompt: (v: string) => void; + setPhase: (v: Phase) => void; + setError: (v: string | null) => void; + setCode: (v: string) => void; + setRole: (v: "attack" | "defense") => void; + setDisplayName: (v: string) => void; + setDescription: (v: string) => void; + setNumSeeds: (v: number) => void; + generate: () => Promise; + reset: () => void; +} + +const AgentContext = createContext(null); + +export function AgentProvider({ children }: { children: ReactNode }) { + const [prompt, setPrompt] = useState(""); + const [phase, setPhase] = useState("idle"); + const [error, setError] = useState(null); + const [code, setCode] = useState(""); + const [role, setRole] = useState<"attack" | "defense">("attack"); + const [displayName, setDisplayName] = useState(""); + const [description, setDescription] = useState(""); + const [numSeeds, setNumSeeds] = useState(0); + + const generate = useCallback(async () => { + if (!prompt.trim()) return; + setPhase("generating"); + setError(null); + try { + const result = await api.generateCode(prompt.trim()); + setCode(result.code); + setRole(result.role as "attack" | "defense"); + setDisplayName(result.display_name); + setDescription(result.description); + setPhase("review"); + } catch (e: unknown) { + setError(e instanceof Error ? e.message : String(e)); + setPhase("idle"); + } + }, [prompt]); + + const reset = useCallback(() => { + setPhase("idle"); + setError(null); + setCode(""); + setDisplayName(""); + setDescription(""); + setNumSeeds(0); + }, []); + + return ( + + {children} + + ); +} + +export function useAgent() { + const ctx = useContext(AgentContext); + if (!ctx) throw new Error("useAgent must be used within AgentProvider"); + return ctx; +} diff --git a/apps/frontend/src/context/BenchContext.tsx b/apps/frontend/src/context/BenchContext.tsx new file mode 100644 index 0000000..24b59aa --- /dev/null +++ b/apps/frontend/src/context/BenchContext.tsx @@ -0,0 +1,75 @@ +import { createContext, useContext, useState, useEffect, useCallback } from "react"; +import type { ReactNode } from "react"; +import { api } from "../api/client"; +import type { BenchJob } from "../api/client"; + +interface BenchContextValue { + prompt: string; + setPrompt: (v: string) => void; + submitting: boolean; + error: string | null; + activeJob: BenchJob | null; + history: BenchJob[]; + submit: () => Promise; + refreshHistory: () => void; +} + +const BenchContext = createContext(null); + +export function BenchProvider({ children }: { children: ReactNode }) { + const [prompt, setPrompt] = useState(""); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const [activeJob, setActiveJob] = useState(null); + const [history, setHistory] = useState([]); + + const refreshHistory = useCallback(() => { + api.listBenchJobs().then(setHistory).catch(() => {}); + }, []); + + useEffect(() => { + refreshHistory(); + }, [refreshHistory]); + + useEffect(() => { + if (!activeJob || activeJob.status === "completed" || activeJob.status === "failed") return; + const interval = setInterval(() => { + api.getBenchJob(activeJob.id).then((j) => { + setActiveJob(j); + if (j.status === "completed" || j.status === "failed") { + refreshHistory(); + } + }); + }, 2000); + return () => clearInterval(interval); + }, [activeJob?.id, activeJob?.status, refreshHistory]); + + const submit = useCallback(async () => { + if (!prompt.trim()) return; + setSubmitting(true); + setError(null); + try { + const job = await api.createBenchJob(prompt.trim()); + setActiveJob(job); + setPrompt(""); + } catch (e: unknown) { + setError(e instanceof Error ? e.message : String(e)); + } finally { + setSubmitting(false); + } + }, [prompt]); + + return ( + + {children} + + ); +} + +export function useBench() { + const ctx = useContext(BenchContext); + if (!ctx) throw new Error("useBench must be used within BenchProvider"); + return ctx; +} diff --git a/apps/frontend/src/pages/Agent.tsx b/apps/frontend/src/pages/Agent.tsx index 77d6721..863867f 100644 --- a/apps/frontend/src/pages/Agent.tsx +++ b/apps/frontend/src/pages/Agent.tsx @@ -3,6 +3,8 @@ import { useNavigate } from "react-router-dom"; import { Send, Loader2, AlertCircle, CheckCircle, ArrowLeft, Play } from "lucide-react"; import { api } from "../api/client"; import type { AgentConfig } from "../api/client"; +import IntensitySelector from "../components/IntensitySelector"; +import { useAgent } from "../context/AgentContext"; const EXAMPLE_PROMPTS = [ "Design an attack that adaptively scales poisoned updates based on the global model's gradient norm to evade norm-based defenses", @@ -11,20 +13,10 @@ const EXAMPLE_PROMPTS = [ "Design a defense that clusters client updates via k-means and only aggregates the largest cluster", ]; -type Phase = "idle" | "generating" | "review" | "submitting"; - export default function Agent() { const navigate = useNavigate(); + const agent = useAgent(); const [config, setConfig] = useState(null); - const [prompt, setPrompt] = useState(""); - const [phase, setPhase] = useState("idle"); - const [error, setError] = useState(null); - - // Review state - const [code, setCode] = useState(""); - const [role, setRole] = useState<"attack" | "defense">("attack"); - const [displayName, setDisplayName] = useState(""); - const [description, setDescription] = useState(""); useEffect(() => { api.getAgentConfig().then(setConfig).catch(() => {}); @@ -32,53 +24,41 @@ export default function Agent() { const noKey = config && !config.has_api_key; - const handleGenerate = async () => { - if (!prompt.trim()) return; - setPhase("generating"); - setError(null); - try { - const result = await api.generateCode(prompt.trim()); - setCode(result.code); - setRole(result.role as "attack" | "defense"); - setDisplayName(result.display_name); - setDescription(result.description); - setPhase("review"); - } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)); - setPhase("idle"); - } - }; + const [versionConflict, setVersionConflict] = useState(false); - const handleApprove = async () => { - setPhase("submitting"); - setError(null); + const handleApprove = async (updateExisting = false) => { + agent.setPhase("submitting"); + agent.setError(null); + setVersionConflict(false); try { const result = await api.createSubmission({ - code, - role, - display_name: displayName.trim() || "Untitled", + code: agent.code, + role: agent.role, + display_name: agent.displayName.trim() || "Untitled", author: "agent", - description: description.trim() || undefined, + description: agent.description.trim() || undefined, + num_seeds: agent.numSeeds || undefined, + update_existing: updateExisting, }); + agent.reset(); navigate(`/submissions/${result.id}`); } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)); - setPhase("review"); + const msg = e instanceof Error ? e.message : String(e); + if (msg.includes("already exists") && msg.includes("update_existing")) { + setVersionConflict(true); + } + agent.setError(msg); + agent.setPhase("review"); } }; - const handleBack = () => { - setPhase("idle"); - setError(null); - }; - // ── Review view ── - if (phase === "review" || phase === "submitting") { + if (agent.phase === "review" || agent.phase === "submitting") { return (