Skip to content

Commit 2d4e5c9

Browse files
piotrlaczkowskipiotrlaczkowski
committed
fix+test: train_next.py pattern fully tested — StackManager fallback
Source changes: src/agentomatic/stacks/manager.py + add get_llm_config_or_default(name, default='default') helper Falls back gracefully when a named profile (e.g. 'rewrite') is absent from the stack; used by training scripts that optionally use a separate rewrite model but must not crash without one. + fix get_settings() docstring (spurious extra backtick) src/agentomatic/cli/templates.py + train.py template: use get_llm_config_or_default('rewrite') for the rewrite model (no crash when stack has only 'default') + pass rewrite_model=rewrite_model to PromptFitterBridge Tests: tests/test_train_script_pattern.py (54 tests, all passing) TestImports (5) Every import statement from the script resolves without error TestHelperFunctions (18) _json_valid, _keyword_score, _precision, _recall, _f1, _opt_composite (empty/valid/missing-keys/non-json/embedded/expected/ non-dict), _model_spec TestStackManagerGetLlmConfigOrDefault (3) - returns named profile when present - falls back to default when missing - raises when fallback also missing TestMetricConstruction (8) LocalJudgeMetric, OptimizeMetricAdapter(name=), CallableMetric, ExactKeyMatchMetric, agents.WeightedMetric with all component types, MetricLoss, CustomMetric, PromptSearchSpace TestPromptFitterBridgeConstruction (5) Full kwargs (local_agent, llm_base_url, llm_api_key, rewrite_model, min_absolute_improvement, concurrency, …), LLMCaller.configure called, rewrite_model forwarded, live-agent fallback TestBaseGraphAgentFitInterface (4) compile(loss=), fit(optimize_mode, optimize_prompt, optimize_params, search_space, callbacks, max_trials), history.history is dict, evaluate returns report.scores dict TestFitResultAccessors (5) history is list[float], generate_fit_report no crash, summary(), apply() writes prompts.json, to_dict() includes score_history TestFullScriptPatternEndToEnd (6) MetricLoss.compute() without crash, full compile→fit→evaluate cycle, fit_result history+report, WeightedMetric→MetricLoss call chain (verifies string args not RunResult), EarlyStopping construction
1 parent 1660038 commit 2d4e5c9

8 files changed

Lines changed: 1348 additions & 6 deletions

File tree

‎.optimize/fit_result_22c6112c130b.json‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
"experiment_id": "22c6112c130b",
3+
"agent": "",
4+
"best_score": 0.0,
5+
"baseline_score": 0.0,
6+
"absolute_improvement": 0.0,
7+
"score_history": [],
8+
"metric_deltas": {
9+
"composite": 0.0
10+
},
11+
"param_suggestions": {},
12+
"best_config": {
13+
"system_prompt": "You are a helpful AI assistant.",
14+
"user_template": null,
15+
"few_shot_examples": [],
16+
"output_contract": null,
17+
"model_params": {
18+
"temperature": 0.0,
19+
"top_p": 0.7,
20+
"max_tokens": 800
21+
},
22+
"rag_params": {},
23+
"tool_params": {},
24+
"model_choice": null,
25+
"fallback_model": null,
26+
"routing_config": {}
27+
},
28+
"baseline_config": {
29+
"system_prompt": "You are a helpful AI assistant.",
30+
"user_template": null,
31+
"few_shot_examples": [],
32+
"output_contract": null,
33+
"model_params": {
34+
"temperature": 0.0,
35+
"top_p": 0.7,
36+
"max_tokens": 800
37+
},
38+
"rag_params": {},
39+
"tool_params": {},
40+
"model_choice": null,
41+
"fallback_model": null,
42+
"routing_config": {}
43+
},
44+
"failure_clusters": [
45+
{
46+
"label": "severe_failures",
47+
"description": "Responses scoring below 0.3 (1 cases)",
48+
"count": 1,
49+
"suggested_fix": "Major prompt revision needed for these query types.",
50+
"severity": 0.9
51+
}
52+
],
53+
"trials": [
54+
{
55+
"round": 1,
56+
"name": "rewrite_000",
57+
"source": "rewrite",
58+
"phase": "minibatch",
59+
"score": 0.0,
60+
"dimensions": {},
61+
"mutation_notes": "Full prompt rewrite at iteration 0. Analysed 1 failures (avg score 0.000) and 1 successes. Context: 0 rounds history, baseline=0.000, current=0.000."
62+
}
63+
],
64+
"suggestions": [
65+
"No configuration changes improved over the baseline."
66+
],
67+
"duration_seconds": 0.02,
68+
"deployment_recommendation": {
69+
"prompt_version": "v2_fit_22c6112c130b",
70+
"confidence": "no_improvement",
71+
"model_params": {
72+
"temperature": 0.0,
73+
"top_p": 0.7,
74+
"max_tokens": 800
75+
},
76+
"deployment_recommendation": {
77+
"rollout": "hold",
78+
"weight": 0.0,
79+
"monitoring_hours": 0
80+
},
81+
"monitoring": {
82+
"metrics": [
83+
"composite"
84+
],
85+
"rollback_threshold": -0.03,
86+
"rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
87+
},
88+
"expected_improvement": 0.0,
89+
"baseline_score": 0.0,
90+
"projected_score": 0.0
91+
}
92+
}

‎.optimize/fit_result_63a30b48776f.json‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
"experiment_id": "63a30b48776f",
3+
"agent": "",
4+
"best_score": 0.0,
5+
"baseline_score": 0.0,
6+
"absolute_improvement": 0.0,
7+
"score_history": [],
8+
"metric_deltas": {
9+
"composite": 0.0
10+
},
11+
"param_suggestions": {},
12+
"best_config": {
13+
"system_prompt": "You are a helpful AI assistant.",
14+
"user_template": null,
15+
"few_shot_examples": [],
16+
"output_contract": null,
17+
"model_params": {
18+
"temperature": 0.0,
19+
"top_p": 0.7,
20+
"max_tokens": 800
21+
},
22+
"rag_params": {},
23+
"tool_params": {},
24+
"model_choice": null,
25+
"fallback_model": null,
26+
"routing_config": {}
27+
},
28+
"baseline_config": {
29+
"system_prompt": "You are a helpful AI assistant.",
30+
"user_template": null,
31+
"few_shot_examples": [],
32+
"output_contract": null,
33+
"model_params": {
34+
"temperature": 0.0,
35+
"top_p": 0.7,
36+
"max_tokens": 800
37+
},
38+
"rag_params": {},
39+
"tool_params": {},
40+
"model_choice": null,
41+
"fallback_model": null,
42+
"routing_config": {}
43+
},
44+
"failure_clusters": [
45+
{
46+
"label": "severe_failures",
47+
"description": "Responses scoring below 0.3 (1 cases)",
48+
"count": 1,
49+
"suggested_fix": "Major prompt revision needed for these query types.",
50+
"severity": 0.9
51+
}
52+
],
53+
"trials": [
54+
{
55+
"round": 1,
56+
"name": "rewrite_000",
57+
"source": "rewrite",
58+
"phase": "minibatch",
59+
"score": 0.0,
60+
"dimensions": {},
61+
"mutation_notes": "Full prompt rewrite at iteration 0. Analysed 1 failures (avg score 0.000) and 1 successes. Context: 0 rounds history, baseline=0.000, current=0.000."
62+
}
63+
],
64+
"suggestions": [
65+
"No configuration changes improved over the baseline."
66+
],
67+
"duration_seconds": 0.01,
68+
"deployment_recommendation": {
69+
"prompt_version": "v2_fit_63a30b48776f",
70+
"confidence": "no_improvement",
71+
"model_params": {
72+
"temperature": 0.0,
73+
"top_p": 0.7,
74+
"max_tokens": 800
75+
},
76+
"deployment_recommendation": {
77+
"rollout": "hold",
78+
"weight": 0.0,
79+
"monitoring_hours": 0
80+
},
81+
"monitoring": {
82+
"metrics": [
83+
"composite"
84+
],
85+
"rollback_threshold": -0.03,
86+
"rollback_instructions": "If 'composite' drops below baseline, rollback to You are a helpful AI assistant... (previous version)."
87+
},
88+
"expected_improvement": 0.0,
89+
"baseline_score": 0.0,
90+
"projected_score": 0.0
91+
}
92+
}

‎.optimize/fitter_report_22c6112c130b.html‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
6+
<title>PromptFitter Report — </title>
7+
<style>
8+
:root { --bg: #0d1117; --surface: #161b22; --border: #30363d; --text: #c9d1d9; --text-dim: #8b949e; }
9+
* { margin: 0; padding: 0; box-sizing: border-box; }
10+
body { background: var(--bg); color: var(--text); font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif; padding: 2rem; max-width: 1200px; margin: 0 auto; line-height: 1.6; }
11+
h1 { font-size: 1.8rem; color: #f0f6fc; margin-bottom: 0.5rem; }
12+
h2 { font-size: 1.3rem; color: #f0f6fc; margin: 2rem 0 1rem; border-bottom: 1px solid var(--border); padding-bottom: 0.5rem; }
13+
.kpi-card { background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 1.25rem; text-align: center; }
14+
table { width: 100%; border-collapse: collapse; background: var(--surface); border-radius: 8px; overflow: hidden; }
15+
th, td { padding: 0.75rem 1rem; text-align: left; border-bottom: 1px solid var(--border); }
16+
th { background: #1c2128; color: var(--text-dim); font-weight: 600; font-size: 0.85rem; text-transform: uppercase; }
17+
td { color: var(--text); }
18+
code { background: #1c2128; padding: 0.2em 0.4em; border-radius: 4px; font-size: 0.9em; }
19+
footer { text-align: center; margin-top: 3rem; padding: 1rem; color: var(--text-dim); font-size: 0.85rem; border-top: 1px solid var(--border); }
20+
</style>
21+
</head>
22+
<body>
23+
<h1>⚡ PromptFitter Report</h1>
24+
<p style="color: #8b949e;">Agent: <strong style="color: #58a6ff;"></strong> | Experiment: <code>22c6112c130b</code> | 2026-07-20 07:39 UTC</p>
25+
26+
27+
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 1rem; margin: 2rem 0;">
28+
<div class="kpi-card">
29+
<div style="color: #8b949e; font-size: 0.85rem;">Baseline Score</div>
30+
<div style="font-size: 2rem; font-weight: 700; color: #f0883e;">0.0000</div>
31+
</div>
32+
<div class="kpi-card">
33+
<div style="color: #8b949e; font-size: 0.85rem;">Best Score</div>
34+
<div style="font-size: 2rem; font-weight: 700; color: #3fb950;">0.0000</div>
35+
</div>
36+
<div class="kpi-card">
37+
<div style="color: #8b949e; font-size: 0.85rem;">Absolute Improvement</div>
38+
<div style="font-size: 2rem; font-weight: 700; color: #f85149;">+0.0000</div>
39+
</div>
40+
<div class="kpi-card">
41+
<div style="color: #8b949e; font-size: 0.85rem;">Relative Improvement</div>
42+
<div style="font-size: 2rem; font-weight: 700; color: #f85149;">+0.0%</div>
43+
</div>
44+
<div class="kpi-card">
45+
<div style="color: #8b949e; font-size: 0.85rem;">Total Trials</div>
46+
<div style="font-size: 2rem; font-weight: 700; color: #58a6ff;">1</div>
47+
</div>
48+
<div class="kpi-card">
49+
<div style="color: #8b949e; font-size: 0.85rem;">Duration</div>
50+
<div style="font-size: 2rem; font-weight: 700; color: #d2a8ff;">0.0s</div>
51+
</div>
52+
</div>
53+
54+
55+
56+
<h2>📊 Metric Deltas</h2>
57+
<table>
58+
<thead><tr><th>Dimension</th><th>Delta</th></tr></thead>
59+
<tbody>
60+
<tr>
61+
<td>composite</td>
62+
<td style="color: #3fb950;">+0.0000</td>
63+
</tr></tbody>
64+
</table>
65+
66+
67+
<h2>🔍 Failure Clusters</h2>
68+
69+
<div style="background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem; margin: 0.5rem 0;">
70+
<strong style="color: #f0883e;">severe_failures</strong>
71+
<span style="color: #8b949e;"> (1 cases)</span>
72+
<p style="color: #c9d1d9; margin: 0.5rem 0;">Responses scoring below 0.3 (1 cases)</p>
73+
<p style="color: #3fb950; font-style: italic;">💡 Major prompt revision needed for these query types.</p>
74+
</div>
75+
76+
77+
<h2>💡 Recommendations</h2>
78+
<ol style="color: #c9d1d9; line-height: 1.8;"><li>No configuration changes improved over the baseline.</li></ol>
79+
80+
81+
<h2>🚀 Deployment Recommendation</h2>
82+
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 1rem; margin: 1rem 0;">
83+
<div class="kpi-card">
84+
<div style="color: #8b949e; font-size: 0.85rem;">Version</div>
85+
<div style="font-size: 1.2rem; font-weight: 700; color: #58a6ff;">v2_fit_22c6112c130b</div>
86+
</div>
87+
<div class="kpi-card">
88+
<div style="color: #8b949e; font-size: 0.85rem;">Confidence</div>
89+
<div style="font-size: 1.5rem; font-weight: 700; color: #f85149;">no_improvement</div>
90+
</div>
91+
<div class="kpi-card">
92+
<div style="color: #8b949e; font-size: 0.85rem;">Rollout Strategy</div>
93+
<div style="font-size: 1.2rem; font-weight: 700; color: #d2a8ff;">hold @ 0%</div>
94+
</div>
95+
<div class="kpi-card">
96+
<div style="color: #8b949e; font-size: 0.85rem;">Monitor</div>
97+
<div style="font-size: 1.5rem; font-weight: 700; color: #c9d1d9;">0h</div>
98+
</div>
99+
</div>
100+
101+
102+
103+
<h2>📝 Prompt Diff</h2>
104+
<div style="background: #0d1117; border: 1px solid #30363d; border-radius: 8px; padding: 1rem; font-family: monospace; font-size: 0.85rem; overflow-x: auto; white-space: pre-wrap;">
105+
106+
</div>
107+
108+
109+
110+
<footer>Generated by Agentomatic PromptFitter — 2026-07-20 07:39 UTC</footer>
111+
</body>
112+
</html>

0 commit comments

Comments
 (0)