Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ PRs welcome! Figaro is meant to be a readable, research-friendly FL platform.
**Phase 1: Solidifying the Agentic Platform**
- [x] **Interactive Agent Planning** — Multi-turn dialogue support for refining experiments, plus visual topology previews (Plan Preview) before execution.
- [x] **Execution Transparency** — Real-time tracking of node-level status during execution and automated natural-language interpretation of results.
- [ ] **Strict Configuration Engine** — Implement strict Pydantic/JSON Schema validation to resolve historical inconsistencies between `config_schema` and underlying algorithms.
- [x] **Strict Configuration Engine** — Implement strict Pydantic/JSON Schema validation to resolve historical inconsistencies between `config_schema` and underlying algorithms.
- [ ] **Advanced Experiment Tracking** — Multi-dimensional search filtering (by metrics, hyperparameters, status) and configuration version control (diffing).

**Phase 2: LLM & LoRA Federated Fine-Tuning**
Expand Down
2 changes: 1 addition & 1 deletion README.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ figaro/
**Phase 1:夯实 Agentic 实验平台**
- [x] **Agent 交互体验升级** —— 支持多轮对话微调实验计划,提供实验执行前的Plan Preview。
- [x] **执行与分析透明化** —— 支持实验节点的实时状态追踪,以及 Agent 驱动的运行结果自动化图表解释。
- [ ] **配置引擎重构** —— 引入基于 Pydantic/JSON Schema 的严格强校验,彻底修复 `config_schema` 与底层算法实现不一致的问题。
- [x] **配置引擎重构** —— 引入基于 Pydantic/JSON Schema 的严格强校验,彻底修复 `config_schema` 与底层算法实现不一致的问题。
- [ ] **高阶实验管理** —— 支持按指标、超参等多维度搜索过滤实验历史,支持配置文件版本控制与 Diff 差异对比。

**Phase 2:LLM / LoRA 联邦微调支持**
Expand Down
91 changes: 83 additions & 8 deletions apps/backend/app/api/v1/endpoints/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from app.services.agent import AgentExperimentRunService, AgentExperimentService, AgentOptimizationHistoryService, AgentRuntimeService, AgentState
from app.services.agent.graph import FederatedAgentGraphBuilder
from app.services.agent.objectives import AgentOptimizationObjective, resolve_objective
from app.services.agent.planning import build_schema_prompt_context, dumps_for_prompt
from app.services.llm import LLMRegistry, LLMService
from app.schemas.agent import AgentPlanPreviewRequest, AgentPlanPreviewResponse, ExperimentPlanPreview, AgentPlanReviseRequest
import uuid
Expand All @@ -49,6 +50,17 @@ async def list_agent_models() -> AgentModelsResponse:
)


@agent_router.get(
"/config/schema",
response_model=dict,
status_code=status.HTTP_200_OK,
summary="Get Agent config schema",
)
async def get_agent_config_schema() -> dict:
"""Return the schema used by Agent planning and editing UI."""
return AgentExperimentService.get_config_schema()


def _build_experiments(history: list) -> list[AgentExperimentSummary]:
"""
Convert internal history snapshots into API experiment summaries.
Expand Down Expand Up @@ -87,13 +99,55 @@ def _build_experiments(history: list) -> list[AgentExperimentSummary]:
return experiments


def _record_value(record, key: str, default=None):
if hasattr(record, key):
return getattr(record, key)
if isinstance(record, dict):
return record.get(key, default)
return default


def _best_record(history: list):
best = None
best_score = None
for record in history:
score = _record_value(record, "score")
if score is None:
continue
try:
numeric_score = float(score)
except (TypeError, ValueError):
continue
if best_score is None or numeric_score > best_score:
best = record
best_score = numeric_score
return best


def _derive_best_payload(snapshot: dict) -> tuple[dict | None, dict | None]:
best_config = snapshot.get("best_config")
best_metrics = snapshot.get("best_metrics")
if best_config is not None and best_metrics is not None:
return best_config, best_metrics

record = _best_record(snapshot.get("experiments", []))
if record is None:
return best_config, best_metrics
if best_config is None:
best_config = _record_value(record, "config")
if best_metrics is None:
best_metrics = _record_value(record, "metrics")
return best_config, best_metrics


def _build_progress_response(snapshot: dict) -> AgentOptimizeProgressResponse:
"""
Convert a runtime snapshot into the progress response schema.
"""
best_config_payload, best_metrics_payload = _derive_best_payload(snapshot)
best_metrics = (
SimulationRunMetricsResponse.model_validate(snapshot["best_metrics"])
if snapshot.get("best_metrics") is not None
SimulationRunMetricsResponse.model_validate(best_metrics_payload)
if best_metrics_payload is not None
else None
)

Expand Down Expand Up @@ -143,10 +197,11 @@ def _build_progress_response(snapshot: dict) -> AgentOptimizeProgressResponse:
completed_iterations=snapshot.get("completed_iterations", 0),
current_plan=current_plan,
current_experiment=current_experiment,
best_config=snapshot.get("best_config"),
best_config=best_config_payload,
best_metrics=best_metrics,
experiments=_build_experiments(snapshot.get("experiments", [])),
draft_experiments=snapshot.get("draft_experiments", []),
config_constraints=snapshot.get("config_constraints", {}),
summary_text=snapshot.get("summary_text"),
error_message=snapshot.get("error_message"),
created_at=snapshot.get("created_at"),
Expand Down Expand Up @@ -222,6 +277,7 @@ async def get_optimization_job(
snapshot.setdefault("current_iteration", job.current_iteration)
snapshot.setdefault("completed_iterations", job.completed_iterations)
snapshot.setdefault("experiments", [])
snapshot.setdefault("config_constraints", {})
snapshot.setdefault("created_at", job.created_at)
snapshot.setdefault("updated_at", job.updated_at)
snapshot.setdefault("finished_at", job.finished_at)
Expand All @@ -248,6 +304,7 @@ async def start_optimization(
model_name=payload.model_name,
objective=payload.objective,
planned_experiments=payload.planned_experiments,
config_constraints=payload.config_constraints,
)
return _build_progress_response(snapshot)

Expand Down Expand Up @@ -304,6 +361,7 @@ async def optimize(
job_name=payload.job_name,
objective=payload.objective,
resolved_objective=resolved_objective,
config_constraints=payload.config_constraints,
)

# LangGraph executor expects a dict-like object; dataclass is fine.
Expand All @@ -318,6 +376,12 @@ async def optimize(
final_state = AgentState(**raw_state) # type: ignore[arg-type]

experiments = _build_experiments(final_state.history)
best = _best_record(final_state.history)
best_metrics = (
SimulationRunMetricsResponse.model_validate(_record_value(best, "metrics"))
if best is not None and _record_value(best, "metrics") is not None
else None
)

return AgentOptimizeResponse(
goal=final_state.goal,
Expand All @@ -326,8 +390,8 @@ async def optimize(
objective=final_state.objective,
resolved_objective=final_state.resolved_objective,
iterations_executed=final_state.iteration,
best_config=None,
best_metrics=None,
best_config=_record_value(best, "config") if best is not None else None,
best_metrics=best_metrics,
experiments=experiments,
summary_text=final_state.summary,
)
Expand Down Expand Up @@ -497,6 +561,7 @@ async def generate_plan_preview(
max_iterations=10,
objective=AgentOptimizationObjective.AUTO,
resolved_objective=AgentOptimizationObjective.ACCURACY,
config_constraints=payload.config_constraints,
)

# Call the graph's parse node directly to get the LLM result
Expand Down Expand Up @@ -528,6 +593,9 @@ async def generate_plan_preview(
"goal": payload.goal,
"job_name": payload.job_name,
"status": "pending_review",
"system_mode": payload.system_mode,
"model_name": payload.model_name,
"config_constraints": payload.config_constraints,
"experiments": [],
"draft_experiments": [p.model_dump() for p in previews]
}
Expand All @@ -537,7 +605,8 @@ async def generate_plan_preview(
optimization_job_id=job.id,
goal=payload.goal,
experiments=previews,
system_mode=payload.system_mode
system_mode=payload.system_mode,
config_constraints=payload.config_constraints,
)

@agent_router.post(
Expand All @@ -560,19 +629,24 @@ async def revise_plan_preview(

snapshot = job.snapshot_json
current_experiments = snapshot.get("draft_experiments", [])
config_constraints = snapshot.get("config_constraints", {})
if not current_experiments:
raise HTTPException(status_code=400, detail="No draft experiments found to revise.")

instructions = (
"You are an AI assistant managing a Federated Learning experiment plan. "
"The user wants to modify the current experiment configuration based on their feedback. "
"Update the JSON configuration to reflect their request. "
"Respect the structured config constraints and do not use disabled future options. "
"Respond ONLY with a valid JSON array of experiment objects, matching the original schema. "
"Do not include any other text."
)
schema_context = build_schema_prompt_context(AgentExperimentService.get_config_schema())

input_text = (
f"Current Plan (JSON):\n{json.dumps(current_experiments, indent=2)}\n\n"
f"Structured Config Constraints (JSON):\n{json.dumps(config_constraints, indent=2)}\n\n"
f"Schema Context (JSON):\n{dumps_for_prompt(schema_context)}\n\n"
f"User Modification Request:\n{payload.instruction}\n\n"
"Please output the updated JSON array of experiments:"
)
Expand Down Expand Up @@ -608,5 +682,6 @@ async def revise_plan_preview(
optimization_job_id=job.id,
goal=snapshot.get("goal", ""),
experiments=[ExperimentPlanPreview.model_validate(exp) for exp in updated_experiments],
system_mode=snapshot.get("system_mode", "simulation")
)
system_mode=snapshot.get("system_mode", "simulation"),
config_constraints=config_constraints,
)
9 changes: 8 additions & 1 deletion apps/backend/app/schemas/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ class AgentOptimizeRequest(BaseModel):
default=None,
description="Explicit list of experiment patches to run, bypassing LLM parse."
)
config_constraints: dict[str, Any] = Field(
default_factory=dict,
description="Structured config constraints selected by the user before planning.",
)


class AgentModelsResponse(BaseModel):
Expand Down Expand Up @@ -148,6 +152,7 @@ class AgentOptimizeProgressResponse(BaseModel):
best_metrics: SimulationRunMetricsResponse | None = None
experiments: list[AgentExperimentSummary] = Field(default_factory=list)
draft_experiments: list[dict[str, Any]] = Field(default_factory=list)
config_constraints: dict[str, Any] = Field(default_factory=dict)
summary_text: str | None = None
error_message: str | None = None
created_at: datetime | None = None
Expand Down Expand Up @@ -237,14 +242,16 @@ class AgentPlanPreviewRequest(BaseModel):
job_name: str = Field(..., description="Name for the job group.")
model_name: Optional[str] = None
system_mode: str = "simulation"
config_constraints: dict[str, Any] = Field(default_factory=dict)

class AgentPlanPreviewResponse(BaseModel):
"""Draft results returned to the frontend."""
optimization_job_id: int
goal: str
experiments: list[ExperimentPlanPreview]
system_mode: str
config_constraints: dict[str, Any] = Field(default_factory=dict)

class AgentPlanReviseRequest(BaseModel):
"""Payload for requesting a revision to an existing plan draft."""
instruction: str = Field(..., description="Natural language feedback to revise the plan.")
instruction: str = Field(..., description="Natural language feedback to revise the plan.")
2 changes: 1 addition & 1 deletion apps/backend/app/services/agent/experiment_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class AgentExperimentService:
"""Application service for agent experiment operations."""

EXPERIMENT_NAME_CONFLICT_MESSAGE = "Experiment name already exists"
_SCHEMA_META_KEYS = {"role", "depends_on", "hidden"}
_SCHEMA_META_KEYS = {"role", "depends_on", "hidden", "ui"}

def __init__(self, session: AsyncSession):
self.session = session
Expand Down
Loading
Loading