V0.2

.gitignore

@@ -1,5 +1,14 @@
 # Datasets
-datasets/
+datasets/*
+!datasets/llm/
+datasets/llm/*
+!datasets/llm/README.md
+
+# Local LLM models
+models/*
+!models/llm/
+models/llm/*
+!models/llm/README.md
 
 # Python
 __pycache__/
@@ -33,4 +42,4 @@ configs/agent_experiment_runs/
 configs/agent_config_*.yaml
 configs/cached/
 
-.pnpm-store/
+.pnpm-store/

README.md

@@ -136,6 +136,21 @@ cd apps/frontend && pnpm install && pnpm dev
 - Frontend: `http://localhost:5173`
 - Backend API: `http://localhost:8000/docs`
 
+### LLM PEFT smoke run
+
+The LLM PEFT smoke config uses a tiny local Hugging Face-compatible model so it can run without downloading a base model. Generate it once before selecting `configs/smoke/llm_peft_simulation.yaml` in the Simulation UI:
+
+```bash
+uv run python scripts/create_tiny_llm_fixture.py
+```
+
+For a direct PowerShell smoke run:
+
+```powershell
+$env:PYTHONPATH="libs;apps/backend/runners"
+uv run python -c "from pathlib import Path; from runtime_dispatcher import run_runtime; raise SystemExit(0 if run_runtime(Path('configs/smoke/llm_peft_simulation.yaml')) else 1)"
+```
+
 ### Distributed deployment
 
 ```bash
@@ -194,8 +209,12 @@ figaro/
 │   ├── models/               # CNN / ResNet
 │   ├── data/                 # Data loading & partitioning
 │   ├── privacy/              # CKKS encryption
-│   └── compression/          # Top-K sparsification
+│   ├── compression/          # Top-K sparsification
+│   └── llm/                  # LLM PEFT runtime utilities
 ├── configs/                  # Experiment configs
+├── datasets/llm/             # Local LLM SFT/evaluation JSONL files
+├── models/llm/               # Local Hugging Face-compatible LLM directories
+├── skill/                    # Local operational skill guides
 ├── scripts/                  # Docker deployment scripts
 └── .github/workflows/        # CI pipelines
 ```
@@ -210,14 +229,16 @@ PRs welcome! Figaro is meant to be a readable, research-friendly FL platform.
 - [x] **Interactive Agent Planning** — Multi-turn dialogue support for refining experiments, plus visual topology previews (Plan Preview) before execution.
 - [x] **Execution Transparency** — Real-time tracking of node-level status during execution and automated natural-language interpretation of results.
 - [x] **Strict Configuration Engine** — Implement strict Pydantic/JSON Schema validation to resolve historical inconsistencies between `config_schema` and underlying algorithms.
-- [ ] **Advanced Experiment Tracking** — Multi-dimensional search filtering (by metrics, hyperparameters, status) and configuration version control (diffing).
-
-**Phase 2: LLM & LoRA Federated Fine-Tuning**
-- [ ] **Native LLM Ecosystem Integration** — Seamless Hugging Face model loading (e.g., Llama 3, Qwen) and efficient parsing of JSONL instruction-tuning datasets.
-- [ ] **Parameter-Efficient Runtime** — Deep integration with LoRA/PEFT, including support for QLoRA (4-bit/8-bit quantization) to lower client-side memory barriers.
-- [ ] **Specialized Adapter Aggregation** — Custom aggregation mechanisms for LoRA adapters, exploring support for heterogeneous LoRA ranks across clients.
-- [ ] **LLM Evaluation Metrics** — Built-in evaluation for generative tasks (Rouge, BLEU, Perplexity) and automated LLM-as-a-Judge capabilities.
-- [ ] **Hardware Guardrails** — Pre-run dynamic GPU memory estimation (OOM prevention) and automated tuning of gradient accumulation and checkpointing.
+- [x] **Advanced Experiment Tracking** — Multi-dimensional search filtering (by metrics, hyperparameters, status) and configuration version control (diffing).
+
+**Phase 2: LLM Federated PEFT Fine-Tuning**
+- [x] **Simulation LoRA/PEFT SFT Route** — `task.type=llm_peft_sft` dispatches to a dedicated single-machine simulation runtime that loads a Hugging Face or local causal LM, applies LoRA adapters, and runs per-client supervised fine-tuning from JSONL data.
+- [x] **LLM/PEFT Configuration Surface** — `config_schema` and runtime normalization now cover base model, tokenizer, max sequence length, precision, SFT dataset path/format, prompt template, LoRA hyperparameters, target modules, quantization mode, and adapter resume path.
+- [x] **JSONL SFT Data Pipeline** — Supports prompt/completion and chat messages JSONL formats, deterministic client splitting, and prompt rendering for plain/chat-style templates.
+- [x] **Adapter-Only Federated Aggregation** — Aggregates LoRA/PEFT adapter tensors by client example count, persists global adapter artifacts, records SHA-256 lineage, and supports warm-starting from a previous global adapter.
+- [x] **LLM Runtime Dependencies & Metrics** — Core project dependencies include `transformers`, `peft`, `accelerate`, `safetensors`, and `bitsandbytes`; backend metrics include train loss, perplexity, token throughput, adapter size, runtime status, dataset summary, and adapter artifact lineage.
+- [x] **Frontend & Agent UX for LLM Runs** — Exposes the LLM route in the schema-driven Simulation and Agent planning flows, with task-aware field visibility, LLM-specific metric charts, local model/dataset selection, and Adapter Lineage artifact views.
+- [x] **Evaluation Harness** — Adds validation JSONL configuration, per-round evaluation loss/perplexity calculation after global adapter aggregation, normalized evaluation metrics, and a generated tiny-model smoke config for LLM PEFT runs.
 
 **Phase 3: Enterprise & Team Collaboration**
 - [ ] **Multi-Tenant Workspaces** — Isolated project environments with Role-Based Access Control (RBAC) and comprehensive audit logging.

README.zh-CN.md

@@ -195,8 +195,12 @@ figaro/
 │   ├── models/               # CNN / ResNet
 │   ├── data/                 # 数据加载与划分
 │   ├── privacy/              # CKKS 加密
-│   └── compression/          # Top-K 稀疏化
+│   ├── compression/          # Top-K 稀疏化
+│   └── llm/                  # LLM PEFT 运行时工具
 ├── configs/                  # 实验配置
+├── datasets/llm/             # 本地 LLM SFT / 评测 JSONL 文件
+├── models/llm/               # 本地 Hugging Face 兼容 LLM 目录
+├── skill/                    # 本地操作说明 skill
 ├── scripts/                  # Docker 部署脚本
 └── .github/workflows/        # CI 流水线
 ```
@@ -211,14 +215,16 @@ figaro/
 - [x] **Agent 交互体验升级** —— 支持多轮对话微调实验计划，提供实验执行前的Plan Preview。
 - [x] **执行与分析透明化** —— 支持实验节点的实时状态追踪，以及 Agent 驱动的运行结果自动化图表解释。
 - [x] **配置引擎重构** —— 引入基于 Pydantic/JSON Schema 的严格强校验，彻底修复 `config_schema` 与底层算法实现不一致的问题。
-- [ ] **高阶实验管理** —— 支持按指标、超参等多维度搜索过滤实验历史，支持配置文件版本控制与 Diff 差异对比。
-
-**Phase 2：LLM / LoRA 联邦微调支持**
-- [ ] **大模型生态原生接入** —— 内置 Hugging Face 适配层，一键加载主流开源模型，支持 JSONL 格式的指令微调数据集高效解析。
-- [ ] **高效分布式微调** —— 深度集成 LoRA/PEFT 训练环境，支持 QLoRA (4-bit/8-bit 量化) 以显著降低边缘节点的显存门槛。
-- [ ] **专属权重聚合策略** —— 针对 LLM 微调定制的 Adapter 权重聚合机制，探索支持客户端异构 LoRA Rank 的聚合方案。
-- [ ] **大模型专项评估体系** —— 集成生成式 NLP 指标，并引入基于大模型的自动化指令跟随能力评估 (LLM-as-a-Judge)。
-- [ ] **资源护栏与预判** —— 训练任务启动前进行动态 GPU 显存预估（防 OOM 机制），并支持梯度累积与 Checkpointing 的自动调优。
+- [x] **高阶实验管理** —— 支持按指标、超参等多维度搜索过滤实验历史，支持配置文件版本控制与 Diff 差异对比。
+
+**Phase 2：基于现有框架的 LLM 联邦 PEFT 微调支持**
+- [x] **单机仿真 LoRA/PEFT SFT 路线** —— `task.type=llm_peft_sft` 会分发到专用的单机仿真运行时，加载 Hugging Face 或本地 Causal LM，注入 LoRA Adapter，并基于 JSONL 数据执行每个客户端的监督微调。
+- [x] **LLM/PEFT 配置面扩展** —— `config_schema` 与运行时规范化已覆盖基础模型、Tokenizer、最大序列长度、精度、SFT 数据路径/格式、Prompt 模板、LoRA 超参、目标模块、量化模式和 Adapter 续跑路径。
+- [x] **JSONL SFT 数据管线** —— 支持 prompt/completion 与 messages 两类 JSONL 格式，支持确定性的客户端数据切分，并提供 plain/chat 风格的文本渲染。
+- [x] **Adapter 权重联邦聚合** —— 按客户端样本数聚合 LoRA/PEFT Adapter Tensor，持久化全局 Adapter 产物，记录 SHA-256 血缘，并支持从历史全局 Adapter warm start 续跑。
+- [x] **LLM 运行依赖与指标** —— 主项目依赖已包含 `transformers`、`peft`、`accelerate`、`safetensors`、`bitsandbytes`；后端指标已包含 train loss、perplexity、token throughput、adapter size、运行状态、数据摘要和 Adapter artifact lineage。
+- [x] **前端与 Agent 的 LLM 运行体验** —— 已在 schema-driven Simulation UI 与 Agent 规划流中开放 LLM 路线，支持按任务类型显示/隐藏字段、LLM 指标曲线、本地模型/数据集选择，以及 Adapter Lineage 产物视图。
+- [x] **评测与 Smoke 配置** —— 已增加验证集 JSONL 配置、全局 Adapter 聚合后的逐轮 evaluation loss/perplexity 计算、标准化评测指标，以及 tiny-model LLM PEFT smoke fixture/config。
 
 **Phase 3：平台化与企业级协作**
 - [ ] **多租户与细粒度权限** —— 构建多用户隔离的项目空间 (Workspaces)，引入基于角色的访问控制 (RBAC) 和完整的操作审计日志。

apps/backend/app/api/v1/endpoints/agent.py

@@ -4,12 +4,16 @@
 
 from __future__ import annotations
 
-from fastapi import APIRouter, status, HTTPException
+from datetime import datetime
+
+from fastapi import APIRouter, status, HTTPException, Query
 import json
 
 from app.api.deps import AsyncSessionDep
 from app.schemas.agent import (
+    AgentConfigDiffResponse,
     AgentConfigChangeResponse,
+    AgentConfigVersionResponse,
     AgentCurrentExperimentResponse,
     AgentCurrentPlanResponse,
     AgentExperimentResponse,
@@ -27,7 +31,14 @@
 from app.services.agent import AgentExperimentRunService, AgentExperimentService, AgentOptimizationHistoryService, AgentRuntimeService, AgentState
 from app.services.agent.graph import FederatedAgentGraphBuilder
 from app.services.agent.objectives import AgentOptimizationObjective, resolve_objective
-from app.services.agent.planning import build_schema_prompt_context, dumps_for_prompt
+from app.services.agent.planning import (
+    build_initial_config,
+    build_schema_prompt_context,
+    deep_merge_config,
+    dumps_for_prompt,
+    lock_structured_constraints,
+    merge_llm_resource_constraints,
+)
 from app.services.llm import LLMRegistry, LLMService
 from app.schemas.agent import AgentPlanPreviewRequest, AgentPlanPreviewResponse, ExperimentPlanPreview, AgentPlanReviseRequest
 import uuid
@@ -241,12 +252,52 @@ def _build_history_summary(item) -> AgentOptimizationJobSummaryResponse:
     status_code=status.HTTP_200_OK,
     summary="List persisted agent optimization jobs",
 )
-async def list_optimization_jobs(session: AsyncSessionDep) -> list[AgentOptimizationJobSummaryResponse]:
+async def list_optimization_jobs(
+    session: AsyncSessionDep,
+    job_status: str | None = Query(default=None, alias="status"),
+    q: str | None = Query(default=None),
+    model_name: str | None = Query(default=None),
+    objective: str | None = Query(default=None),
+    best_score_min: float | None = Query(default=None),
+    best_score_max: float | None = Query(default=None),
+    created_from: datetime | None = Query(default=None),
+    created_to: datetime | None = Query(default=None),
+    dataset: str | None = Query(default=None),
+    config_model: str | None = Query(default=None),
+    aggregation: str | None = Query(default=None),
+    num_clients: int | None = Query(default=None),
+    num_rounds: int | None = Query(default=None),
+    dataset_name_alias: str | None = Query(default=None, alias="dataset.name"),
+    config_model_alias: str | None = Query(default=None, alias="model.name"),
+    aggregation_alias: str | None = Query(default=None, alias="federated.aggregation"),
+    num_clients_alias: int | None = Query(default=None, alias="federated.num_clients"),
+    num_rounds_alias: int | None = Query(default=None, alias="federated.num_rounds"),
+) -> list[AgentOptimizationJobSummaryResponse]:
     """
     Return persisted Agent optimization jobs ordered by latest update time.
     """
     service = AgentOptimizationHistoryService(session)
-    jobs = await service.list_jobs()
+    config_filters = {
+        "dataset.name": dataset_name_alias or dataset,
+        "model.name": config_model_alias or config_model,
+        "federated.aggregation": aggregation_alias or aggregation,
+        "federated.num_clients": num_clients_alias if num_clients_alias is not None else num_clients,
+        "federated.num_rounds": num_rounds_alias if num_rounds_alias is not None else num_rounds,
+    }
+    try:
+        jobs = await service.list_jobs(
+            status=job_status,
+            q=q,
+            model_name=model_name,
+            objective=objective,
+            best_score_min=best_score_min,
+            best_score_max=best_score_max,
+            created_from=created_from,
+            created_to=created_to,
+            config_filters=config_filters,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
     return [_build_history_summary(item) for item in jobs]
 
 
@@ -284,6 +335,67 @@ async def get_optimization_job(
     return _build_progress_response(snapshot)
 
 
+def _to_config_version_response(version) -> AgentConfigVersionResponse:
+    return AgentConfigVersionResponse(
+        id=version.id,
+        optimization_job_id=version.optimization_job_id,
+        run_id=version.run_id,
+        iteration=version.iteration,
+        source=version.source,
+        label=version.label,
+        config_hash=version.config_hash,
+        config_json=version.config_json if isinstance(version.config_json, dict) else {},
+        diff_json=[
+            AgentConfigChangeResponse.model_validate(item)
+            for item in (version.diff_json if isinstance(version.diff_json, list) else [])
+        ],
+        created_at=version.created_at,
+    )
+
+
+@agent_router.get(
+    "/optimization-jobs/{optimization_job_id}/config-versions",
+    response_model=list[AgentConfigVersionResponse],
+    status_code=status.HTTP_200_OK,
+    summary="List persisted config versions for an agent optimization job",
+)
+async def list_optimization_job_config_versions(
+    optimization_job_id: int,
+    session: AsyncSessionDep,
+) -> list[AgentConfigVersionResponse]:
+    """Return config versions captured during planning and experiment execution."""
+    service = AgentOptimizationHistoryService(session)
+    versions = await service.list_config_versions(optimization_job_id)
+    return [_to_config_version_response(version) for version in versions]
+
+
+@agent_router.get(
+    "/optimization-jobs/{optimization_job_id}/config-diff",
+    response_model=AgentConfigDiffResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Diff two persisted config versions for an agent optimization job",
+)
+async def diff_optimization_job_config_versions(
+    optimization_job_id: int,
+    session: AsyncSessionDep,
+    to_version_id: int = Query(...),
+    from_version_id: int | None = Query(default=None),
+) -> AgentConfigDiffResponse:
+    """Return a normalized config diff between two persisted versions."""
+    service = AgentOptimizationHistoryService(session)
+    changes = await service.diff_config_versions(
+        optimization_job_id=optimization_job_id,
+        from_version_id=from_version_id,
+        to_version_id=to_version_id,
+    )
+    return AgentConfigDiffResponse(
+        optimization_job_id=optimization_job_id,
+        from_version_id=from_version_id,
+        to_version_id=to_version_id,
+        changes=[AgentConfigChangeResponse.model_validate(item) for item in changes],
+    )
+
+
 @agent_router.post(
     "/optimize/start",
     response_model=AgentOptimizeProgressResponse,
@@ -675,13 +787,39 @@ async def revise_plan_preview(
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Failed to parse LLM response: {str(e)}")
 
-    snapshot["draft_experiments"] = updated_experiments
+    experiment_service = AgentExperimentService(session)
+    schema = experiment_service.get_config_schema()
+    constrained_base = experiment_service.normalize_simulation_config(
+        deep_merge_config(build_initial_config(schema), config_constraints)
+    )
+    effective_constraints = merge_llm_resource_constraints(constrained_base, config_constraints)
+    normalized_experiments = []
+    for idx, exp in enumerate(updated_experiments):
+        if not isinstance(exp, dict):
+            continue
+        raw_config = exp.get("config_patch", exp.get("config", {}))
+        if not isinstance(raw_config, dict):
+            raw_config = {}
+        merged = lock_structured_constraints(
+            deep_merge_config(constrained_base, raw_config),
+            effective_constraints,
+        )
+        normalized_experiments.append(
+            {
+                **exp,
+                "name": exp.get("name", f"exp-{idx + 1}"),
+                "plan_summary": exp.get("plan_summary", snapshot.get("goal", "")),
+                "config_patch": experiment_service.normalize_simulation_config(merged),
+            }
+        )
+
+    snapshot["draft_experiments"] = normalized_experiments
     await history_service.update_job_snapshot(task_id=job.task_id, snapshot=snapshot)
 
     return AgentPlanPreviewResponse(
         optimization_job_id=job.id,
         goal=snapshot.get("goal", ""),
-        experiments=[ExperimentPlanPreview.model_validate(exp) for exp in updated_experiments],
+        experiments=[ExperimentPlanPreview.model_validate(exp) for exp in normalized_experiments],
         system_mode=snapshot.get("system_mode", "simulation"),
         config_constraints=config_constraints,
     )

apps/backend/app/models/__init__.py

@@ -1,8 +1,10 @@
 """Model package marker — import all models so SQLModel registers them."""
 
 from app.models.agent import (  # noqa: F401
+    AgentConfigVersion,
     AgentExperiment,
     AgentExperimentRun,
     AgentExperimentRunLog,
     AgentExperimentRunResult,
+    AgentOptimizationJob,
 )

apps/backend/app/models/agent/__init__.py

@@ -10,7 +10,7 @@
     AgentExperimentRunStatus,
     AgentExperimentStatus,
 )
-from app.models.agent.optimization_jobs import AgentOptimizationJob, AgentOptimizationJobStatus
+from app.models.agent.optimization_jobs import AgentConfigVersion, AgentOptimizationJob, AgentOptimizationJobStatus
 
 __all__ = [
     "AgentExperiment",
@@ -21,4 +21,5 @@
     "AgentExperimentRunResult",
     "AgentOptimizationJob",
     "AgentOptimizationJobStatus",
+    "AgentConfigVersion",
 ]