Skip to content

feat: add error handling and status updates for scenario runs #431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bayger merged 1 commit into from
Jun 18, 2026
Merged

feat: add error handling and status updates for scenario runs #431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 50 additions & 2 deletions src/api/generated/Api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13599,6 +13599,11 @@ export class Api<
status: ScenarioRunStatus;
/** Human-readable details about the current status, e.g. failure reason or cancellation actor */
statusDetails: string | null;
/**
* Number of conversations that errored during execution (excluded from pass/fail evaluation)
* @min 0
*/
errorCount: number;
/** Additional metadata */
metadata: Record<string, any>;
/** Version number for optimistic locking */
Expand Down Expand Up @@ -13687,6 +13692,11 @@ export class Api<
status: ScenarioRunStatus;
/** Human-readable details about the current status, e.g. failure reason or cancellation actor */
statusDetails: string | null;
/**
* Number of conversations that errored during execution (excluded from pass/fail evaluation)
* @min 0
*/
errorCount: number;
/** Additional metadata */
metadata: Record<string, any>;
/** Version number for optimistic locking */
Expand Down Expand Up @@ -13760,6 +13770,11 @@ export class Api<
status: ScenarioRunStatus;
/** Human-readable details about the current status, e.g. failure reason or cancellation actor */
statusDetails: string | null;
/**
* Number of conversations that errored during execution (excluded from pass/fail evaluation)
* @min 0
*/
errorCount: number;
/** Additional metadata */
metadata: Record<string, any>;
/** Version number for optimistic locking */
Expand Down Expand Up @@ -13833,6 +13848,11 @@ export class Api<
status: ScenarioRunStatus;
/** Human-readable details about the current status, e.g. failure reason or cancellation actor */
statusDetails: string | null;
/**
* Number of conversations that errored during execution (excluded from pass/fail evaluation)
* @min 0
*/
errorCount: number;
/** Additional metadata */
metadata: Record<string, any>;
/** Version number for optimistic locking */
Expand Down Expand Up @@ -13974,7 +13994,21 @@ export class Api<
/** ID of the underlying conversation used to run this scenario conversation */
conversationId: string | null;
/** Current execution status of this conversation */
status: "queued" | "in_progress" | "passed" | "failed" | "cancelled";
status:
| "queued"
| "in_progress"
| "passed"
| "failed"
| "cancelled"
| "error";
/** How the test conversation ended */
testRunStatus:
| "conversation_ended"
| "conversation_aborted"
| "conversation_failed"
| "max_turns_reached"
| "tester_hung_up"
| null;
/** Extracted stage variable values at the end of the conversation */
dataExtractionResults: Record<string, any>;
/** Post-processed data transformation results */
Expand Down Expand Up @@ -14051,7 +14085,21 @@ export class Api<
/** ID of the underlying conversation used to run this scenario conversation */
conversationId: string | null;
/** Current execution status of this conversation */
status: "queued" | "in_progress" | "passed" | "failed" | "cancelled";
status:
| "queued"
| "in_progress"
| "passed"
| "failed"
| "cancelled"
| "error";
/** How the test conversation ended */
testRunStatus:
| "conversation_ended"
| "conversation_aborted"
| "conversation_failed"
| "max_turns_reached"
| "tester_hung_up"
| null;
/** Extracted stage variable values at the end of the conversation */
dataExtractionResults: Record<string, any>;
/** Post-processed data transformation results */
Expand Down
43 changes: 41 additions & 2 deletions src/api/generated/data-contracts.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export enum ScenarioRunStatus {
Passed = "passed",
Failed = "failed",
Cancelled = "cancelled",
Error = "error",
}

/** Tool execution type: smart_function (LLM-based), webhook (HTTP call), script (JavaScript) */
Expand Down Expand Up @@ -9022,6 +9023,11 @@ export interface ScenarioRunResponse {
status: ScenarioRunStatus;
/** Human-readable details about the current status, e.g. failure reason or cancellation actor */
statusDetails: string | null;
/**
* Number of conversations that errored during execution (excluded from pass/fail evaluation)
* @min 0
*/
errorCount: number;
/** Additional metadata */
metadata: Record<string, any>;
/** Version number for optimistic locking */
Expand Down Expand Up @@ -9055,6 +9061,11 @@ export interface ScenarioRunListResponse {
status: ScenarioRunStatus;
/** Human-readable details about the current status, e.g. failure reason or cancellation actor */
statusDetails: string | null;
/**
* Number of conversations that errored during execution (excluded from pass/fail evaluation)
* @min 0
*/
errorCount: number;
/** Additional metadata */
metadata: Record<string, any>;
/** Version number for optimistic locking */
Expand Down Expand Up @@ -9104,7 +9115,21 @@ export interface ScenarioConversationResponse {
/** ID of the underlying conversation used to run this scenario conversation */
conversationId: string | null;
/** Current execution status of this conversation */
status: "queued" | "in_progress" | "passed" | "failed" | "cancelled";
status:
| "queued"
| "in_progress"
| "passed"
| "failed"
| "cancelled"
| "error";
/** How the test conversation ended */
testRunStatus:
| "conversation_ended"
| "conversation_aborted"
| "conversation_failed"
| "max_turns_reached"
| "tester_hung_up"
| null;
/** Extracted stage variable values at the end of the conversation */
dataExtractionResults: Record<string, any>;
/** Post-processed data transformation results */
Expand Down Expand Up @@ -9141,7 +9166,21 @@ export interface ScenarioConversationListResponse {
/** ID of the underlying conversation used to run this scenario conversation */
conversationId: string | null;
/** Current execution status of this conversation */
status: "queued" | "in_progress" | "passed" | "failed" | "cancelled";
status:
| "queued"
| "in_progress"
| "passed"
| "failed"
| "cancelled"
| "error";
/** How the test conversation ended */
testRunStatus:
| "conversation_ended"
| "conversation_aborted"
| "conversation_failed"
| "max_turns_reached"
| "tester_hung_up"
| null;
/** Extracted stage variable values at the end of the conversation */
dataExtractionResults: Record<string, any>;
/** Post-processed data transformation results */
Expand Down
Loading
Loading