Skip to content

feat(flow-chat): expose retry attempt diagnostics #1709

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wsp1911 merged 1 commit into from
Jul 23, 2026
Merged

feat(flow-chat): expose retry attempt diagnostics #1709

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/apps/desktop/src/api/agentic_api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3216,6 +3216,7 @@ mod tests {
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: None,
token_details: None,
status: "completed".to_string(),
Expand Down Expand Up @@ -3297,6 +3298,7 @@ mod tests {
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: None,
token_details: None,
status: "completed".to_string(),
Expand Down Expand Up @@ -3359,6 +3361,7 @@ mod tests {
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: None,
token_details: None,
status: "completed".to_string(),
Expand Down
2 changes: 2 additions & 0 deletions src/crates/assembly/core/src/agentic/coordination/coordinator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1229,6 +1229,7 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: None,
token_details: None,
status: "completed".to_string(),
Expand Down Expand Up @@ -1305,6 +1306,7 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: Some("context_compression".to_string()),
token_details: None,
status: "error".to_string(),
Expand Down
5 changes: 5 additions & 0 deletions src/crates/assembly/core/src/agentic/core/message.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,7 @@ mod tests {
arguments: json!({ "path": "src/main.rs" }),
raw_arguments: Some(r#"{"path":"src/main.rs" "line_end":4}"#.to_string()),
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: ToolArgumentRepairKind::PermissiveNormalToolJsonRepair,
});
Expand All @@ -759,6 +760,9 @@ pub struct ToolCall {
/// Record whether tool parameters are valid
#[serde(default)]
pub is_error: bool,
/// Original JSON parser error when the provider emitted invalid arguments.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parse_error: Option<String>,
/// True when the raw JSON arguments were truncated mid-stream and we
/// successfully repaired them. Downstream consumers can flag this to the
/// model so it understands the content may be incomplete.
Expand Down Expand Up @@ -786,6 +790,7 @@ impl From<bitfun_agent_stream::ToolCall> for ToolCall {
arguments: tool_call.arguments,
raw_arguments: tool_call.raw_arguments,
is_error: tool_call.is_error,
parse_error: tool_call.parse_error,
recovered_from_truncation: tool_call.recovered_from_truncation,
repair_kind: tool_call.repair_kind,
}
Expand Down
3 changes: 2 additions & 1 deletion src/crates/assembly/core/src/agentic/events/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ pub use bitfun_events::agentic::ErrorCategory;
pub use bitfun_events::{
AgenticEvent as BaseAgenticEvent, AgenticEventEnvelope as EventEnvelope,
AgenticEventPriority as EventPriority, DeepReviewQueueReason, DeepReviewQueueState,
DeepReviewQueueStatus, SubagentParentInfo, ToolEventData,
DeepReviewQueueStatus, ModelRoundAttemptDiagnostic, ModelRoundAttemptToolDiagnostic,
SubagentParentInfo, ToolEventData,
};

// ============ Core layer AgenticEvent extension ============
Expand Down
2 changes: 2 additions & 0 deletions src/crates/assembly/core/src/agentic/execution/execution_engine.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4428,6 +4428,7 @@ mod tests {
arguments: json!({ "job_id": "job-1" }),
raw_arguments: None,
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: Default::default(),
}];
Expand Down Expand Up @@ -4456,6 +4457,7 @@ mod tests {
arguments: json!({ "path": "missing.txt" }),
raw_arguments: None,
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: Default::default(),
}];
Expand Down
150 changes: 148 additions & 2 deletions src/crates/assembly/core/src/agentic/execution/round_executor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ use super::model_exchange_trace::prepare_model_exchange_trace;
use super::stream_processor::{StreamProcessOptions, StreamProcessor, StreamResult};
use super::types::{FinishReason, RoundContext, RoundResult};
use crate::agentic::core::{Message, ToolCall};
use crate::agentic::events::{AgenticEvent, EventPriority, EventQueue, ToolEventData};
use crate::agentic::events::{
AgenticEvent, EventPriority, EventQueue, ModelRoundAttemptDiagnostic,
ModelRoundAttemptToolDiagnostic, ToolEventData,
};
use crate::agentic::memories::{
parse_bitfun_memory_citation, parse_bitfun_memory_citation_payloads,
strip_bitfun_memory_citations,
Expand Down Expand Up @@ -62,6 +65,56 @@ impl RoundExecutor {
!text.trim().is_empty()
}

fn retry_diagnostic(
attempt_id: String,
attempt_index: u32,
category: &str,
raw_error: Option<String>,
tool_calls: &[ToolCall],
) -> ModelRoundAttemptDiagnostic {
ModelRoundAttemptDiagnostic {
attempt_id,
attempt_index,
category: category.to_string(),
raw_error,
tool_calls: tool_calls
.iter()
.filter(|tool_call| !tool_call.is_valid())
.map(|tool_call| ModelRoundAttemptToolDiagnostic {
tool_id: (!tool_call.tool_id.is_empty()).then(|| tool_call.tool_id.clone()),
tool_name: (!tool_call.tool_name.is_empty())
.then(|| tool_call.tool_name.clone()),
raw_arguments: tool_call.raw_arguments.clone(),
validation_error: tool_call.parse_error.clone(),
})
.collect(),
}
}

async fn record_retry_diagnostic(
&self,
context: &RoundContext,
round_id: &str,
attempt_id: String,
attempt_index: u32,
category: &str,
raw_error: Option<String>,
tool_calls: &[ToolCall],
) {
let diagnostic =
Self::retry_diagnostic(attempt_id, attempt_index, category, raw_error, tool_calls);
self.emit_event(
AgenticEvent::ModelRoundAttemptSuperseded {
session_id: context.session_id.clone(),
turn_id: context.dialog_turn_id.clone(),
round_id: round_id.to_string(),
diagnostic: diagnostic.clone(),
},
EventPriority::High,
)
.await;
}

fn parsed_memory_citation_from_stream_result(
stream_result: &StreamResult,
) -> Option<crate::agentic::core::message::MemoryCitation> {
Expand Down Expand Up @@ -248,6 +301,16 @@ impl RoundExecutor {
if Self::is_transient_network_error(&err_msg)
&& attempt_index < max_attempts - 1
{
self.record_retry_diagnostic(
&context,
&round_id,
attempt_id.clone(),
attempt_number,
"transient_request_error",
Some(err_msg.clone()),
&[],
)
.await;
let delay_ms = Self::retry_delay_ms_for_error(attempt_index, &err_msg);
warn!(
"Retrying AI request after connection failure: session_id={}, round_id={}, attempt={}/{}, delay_ms={}, error={}",
Expand Down Expand Up @@ -347,6 +410,16 @@ impl RoundExecutor {
&& attempt_index < max_attempts - 1
&& Self::is_transient_network_error(&err_msg)
{
self.record_retry_diagnostic(
&context,
&round_id,
attempt_id.clone(),
attempt_number,
"interrupted_tool_arguments",
Some(err_msg.clone()),
&result.tool_calls,
)
.await;
Self::complete_model_exchange_trace(
trace_config.as_ref(),
trace_handle.as_ref(),
Expand Down Expand Up @@ -438,6 +511,16 @@ impl RoundExecutor {
&& Self::is_transient_network_error(partial_recovery_reason)
&& attempt_index < max_attempts - 1
{
self.record_retry_diagnostic(
&context,
&round_id,
attempt_id.clone(),
attempt_number,
"partial_stream_error",
Some(partial_recovery_reason.to_string()),
&result.tool_calls,
)
.await;
Self::complete_model_exchange_trace(
trace_config.as_ref(),
trace_handle.as_ref(),
Expand All @@ -464,6 +547,16 @@ impl RoundExecutor {
if Self::is_invalid_tool_only_without_text(&result) {
let err_msg = "Provider returned only invalid tool arguments".to_string();
if attempt_index < max_attempts - 1 {
self.record_retry_diagnostic(
&context,
&round_id,
attempt_id.clone(),
attempt_number,
"invalid_tool_arguments",
None,
&result.tool_calls,
)
.await;
Self::complete_model_exchange_trace(
trace_config.as_ref(),
trace_handle.as_ref(),
Expand Down Expand Up @@ -513,6 +606,16 @@ impl RoundExecutor {
}

if no_effective_output && attempt_index < max_attempts - 1 {
self.record_retry_diagnostic(
&context,
&round_id,
attempt_id.clone(),
attempt_number,
"no_effective_output",
None,
&[],
)
.await;
Self::complete_model_exchange_trace(
trace_config.as_ref(),
trace_handle.as_ref(),
Expand Down Expand Up @@ -569,6 +672,16 @@ impl RoundExecutor {
)
.await;
if can_retry {
self.record_retry_diagnostic(
&context,
&round_id,
attempt_id.clone(),
attempt_number,
"transient_stream_error",
Some(err_msg.clone()),
&[],
)
.await;
let delay_ms = Self::retry_delay_ms_for_error(attempt_index, &err_msg);
warn!(
"Retrying stream after transient error with no effective output: session_id={}, round_id={}, attempt={}/{}, delay_ms={}, error={}",
Expand Down Expand Up @@ -1680,6 +1793,7 @@ mod tests {
arguments: json!({}),
raw_arguments: Some("{\"command\":".to_string()),
is_error: true,
parse_error: Some("EOF while parsing an object".to_string()),
recovered_from_truncation: false,
repair_kind: Default::default(),
}],
Expand Down Expand Up @@ -1736,11 +1850,43 @@ mod tests {
"tool_name": "Bash",
"arguments": {},
"raw_arguments": "{\"command\":",
"is_error": true
"is_error": true,
"parse_error": "EOF while parsing an object"
}]))
);
}

#[test]
fn retry_diagnostic_preserves_invalid_tool_arguments_and_parser_error() {
let diagnostic = RoundExecutor::retry_diagnostic(
"round-1:attempt:1".to_string(),
1,
"invalid_tool_arguments",
None,
&[ToolCall {
tool_id: "tool-1".to_string(),
tool_name: "Bash".to_string(),
arguments: json!({}),
raw_arguments: Some("{\"command\":".to_string()),
is_error: true,
parse_error: Some("EOF while parsing an object".to_string()),
recovered_from_truncation: false,
repair_kind: Default::default(),
}],
);

assert_eq!(diagnostic.category, "invalid_tool_arguments");
assert_eq!(diagnostic.tool_calls.len(), 1);
assert_eq!(
diagnostic.tool_calls[0].raw_arguments.as_deref(),
Some("{\"command\":")
);
assert_eq!(
diagnostic.tool_calls[0].validation_error.as_deref(),
Some("EOF while parsing an object")
);
}

#[test]
fn error_trace_response_without_stream_result_stays_empty() {
let trace = RoundExecutor::error_trace_response("error", "request failed".to_string());
Expand Down
2 changes: 2 additions & 0 deletions src/crates/assembly/core/src/agentic/insights/collector.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,6 +658,7 @@ fn rebuild_messages_from_turns(turns: &[DialogTurnData]) -> Vec<Message> {
arguments: ti.tool_call.input.clone(),
raw_arguments: None,
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: Default::default(),
})
Expand Down Expand Up @@ -1129,6 +1130,7 @@ mod tests {
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: None,
token_details: None,
status: "completed".to_string(),
Expand Down
1 change: 1 addition & 0 deletions src/crates/assembly/core/src/agentic/memories/transcript.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,7 @@ mod tests {
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: None,
token_details: None,
status: "completed".to_string(),
Expand Down
1 change: 1 addition & 0 deletions src/crates/assembly/core/src/agentic/persistence/manager.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3438,6 +3438,7 @@ mod tests {
first_visible_output_ms: None,
stream_duration_ms: None,
attempt_count: None,
attempt_diagnostics: vec![],
failure_category: None,
token_details: None,
status: "completed".to_string(),
Expand Down
1 change: 1 addition & 0 deletions src/crates/assembly/core/src/agentic/session/compression/compressor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,6 +611,7 @@ mod tests {
}),
raw_arguments: None,
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: Default::default(),
}],
Expand Down
3 changes: 3 additions & 0 deletions src/crates/assembly/core/src/agentic/session/compression/fallback/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ fn clears_tool_results_from_compressed_history() {
}),
raw_arguments: None,
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: Default::default(),
}],
Expand Down Expand Up @@ -166,6 +167,7 @@ fn groups_consecutive_assistant_messages_under_single_role_header() {
}),
raw_arguments: None,
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: Default::default(),
}],
Expand All @@ -182,6 +184,7 @@ fn groups_consecutive_assistant_messages_under_single_role_header() {
}),
raw_arguments: None,
is_error: false,
parse_error: None,
recovered_from_truncation: false,
repair_kind: Default::default(),
}],
Expand Down
Loading
Loading