Skip to content

Add deterministic workflow run metadata #339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chubes4 merged 4 commits into from
Jun 18, 2026
Merged

Add deterministic workflow run metadata #339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5c8700a
Add deterministic workflow run metadata
chubes4 Jun 18, 2026
2946f9e
Fix workflow lint formatting
chubes4 Jun 18, 2026
5882ccd
Merge origin/main into workflow run record branch
chubes4 Jun 18, 2026
82bc736
Merge latest main into workflow run record branch
chubes4 Jun 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions src/Workflows/class-wp-agent-workflow-run-result.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ final class WP_Agent_Workflow_Run_Result {
* @param int $started_at Unix timestamp.
* @param int $ended_at Unix timestamp; 0 while running.
* @param array<mixed> $metadata Free-form metadata for recorders / tracers (Langfuse trace ids, etc.).
* @param array<mixed> $evidence_refs Neutral JSON-serializable artifact/log references owned by the host.
* @param array<mixed> $evidence_refs Neutral JSON-serializable artifact/log references owned by the host.
* @param array<mixed> $replay_metadata Deterministic metadata needed to replay or audit this run.
*/
public function __construct(
private string $run_id,
Expand All @@ -61,14 +62,15 @@ public function __construct(
private int $started_at,
private int $ended_at,
private array $metadata,
private array $evidence_refs = array()
private array $evidence_refs = array(),
private array $replay_metadata = array()
) {}

/**
* @param array<mixed> $inputs
*/
public static function pending( string $run_id, string $workflow_id, array $inputs, int $started_at ): self {
return new self( $run_id, $workflow_id, self::STATUS_PENDING, $inputs, array(), array(), array(), $started_at, 0, array(), array() );
return new self( $run_id, $workflow_id, self::STATUS_PENDING, $inputs, array(), array(), array(), $started_at, 0, array(), array(), array() );
}

/**
Expand All @@ -91,7 +93,8 @@ public static function from_array( array $value ): self {
self::int_value( $value['started_at'] ?? 0 ),
self::int_value( $value['ended_at'] ?? 0 ),
self::array_value( $value['metadata'] ?? array() ),
self::array_value( $value['evidence_refs'] ?? array() )
self::array_value( $value['evidence_refs'] ?? array() ),
self::array_value( $value['replay'] ?? array() )
);
}

Expand Down Expand Up @@ -157,6 +160,13 @@ public function get_evidence_refs(): array {
return $this->evidence_refs;
}

/**
* @return array<mixed>
*/
public function get_replay_metadata(): array {
return $this->replay_metadata;
}

public function is_succeeded(): bool {
return self::STATUS_SUCCEEDED === $this->status;
}
Expand Down Expand Up @@ -187,6 +197,7 @@ public function with( array $patch ): self {
self::int_patch_value( $patch, 'ended_at', $this->ended_at ),
self::array_patch_value( $patch, 'metadata', $this->metadata ),
self::array_patch_value( $patch, 'evidence_refs', $this->evidence_refs ),
self::array_patch_value( $patch, 'replay', $this->replay_metadata ),
);
}

Expand Down Expand Up @@ -249,6 +260,7 @@ public function to_array(): array {
'ended_at' => $this->ended_at,
'metadata' => $this->metadata,
'evidence_refs' => $this->evidence_refs,
'replay' => $this->replay_metadata,
);
}
}
59 changes: 58 additions & 1 deletion src/Workflows/class-wp-agent-workflow-runner.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ public function run( WP_Agent_Workflow_Spec $spec, array $inputs = array(), arra
$run_id = self::string_value( $options['run_id'] ?? self::generate_run_id() );
$metadata = (array) ( $options['metadata'] ?? array() );
$evidence_refs = (array) ( $options['evidence_refs'] ?? array() );
$replay = self::build_replay_metadata( $spec );

// Build the initial RUNNING result and persist via recorder->start()
// before doing anything else. Even if input validation fails on the
Expand All @@ -121,7 +122,8 @@ public function run( WP_Agent_Workflow_Spec $spec, array $inputs = array(), arra
$started_at,
0,
$metadata,
$evidence_refs
$evidence_refs,
$replay
);

if ( $this->recorder ) {
Expand Down Expand Up @@ -250,6 +252,61 @@ private static function validate_inputs( WP_Agent_Workflow_Spec $spec, array $in
return null;
}

/**
* Build deterministic metadata for recorders and replay tooling.
*
* @return array<string, mixed>
*/
private static function build_replay_metadata( WP_Agent_Workflow_Spec $spec ): array {
$spec_snapshot = $spec->to_array();

return array(
'run_record_schema_version' => 1,
'workflow_spec_version' => $spec->get_version(),
'workflow_spec_hash' => hash( 'sha256', self::canonical_json( $spec_snapshot ) ),
'workflow_spec_snapshot' => $spec_snapshot,
);
}

/**
* JSON encode arrays with sorted object keys so equivalent specs hash the same.
*
* @param mixed $value Value to encode.
*/
private static function canonical_json( $value ): string {
$normalized = self::sort_recursive( $value );

// phpcs:ignore WordPress.WP.AlternativeFunctions.json_encode_json_encode -- Pure-PHP smoke tests run without WordPress loaded.
$encoded = function_exists( 'wp_json_encode' ) ? wp_json_encode( $normalized ) : json_encode( $normalized );

if ( false === $encoded ) {
return '';
}

return $encoded;
}

/**
* @param mixed $value Value to sort.
* @return mixed
*/
private static function sort_recursive( $value ) {
if ( ! is_array( $value ) ) {
return $value;
}

$is_list = array_keys( $value ) === range( 0, count( $value ) - 1 );
if ( ! $is_list ) {
ksort( $value );
}

foreach ( $value as $key => $child ) {
$value[ $key ] = self::sort_recursive( $child );
}

return $value;
}

/**
* Default `ability` step handler: invokes a registered Abilities API
* ability with the step's `args` (post-binding-resolution). Returns
Expand Down
43 changes: 42 additions & 1 deletion src/Workflows/class-wp-agent-workflow-step-executor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ public function execute( array $step, WP_Agent_Workflow_Run_Context $context ):
'ended_at' => 0,
);

$handler = $this->handlers[ $type ] ?? null;
$handler = $this->handlers[ $type ] ?? null;
$record['handler'] = self::describe_handler( $handler );
if ( ! is_callable( $handler ) ) {
$record['status'] = WP_Agent_Workflow_Run_Result::STATUS_SKIPPED;
$record['ended_at'] = time();
Expand All @@ -53,6 +54,7 @@ public function execute( array $step, WP_Agent_Workflow_Run_Context $context ):
$resolved = 'foreach' === $type
? self::expand_foreach_outer_step( $step, $context_array )
: WP_Agent_Workflow_Bindings::expand( $step, $context_array );
$record['resolved_step'] = is_array( $resolved ) ? $resolved : array();
$handler_context = $context_array;
$handler_context['_workflow_step_handlers'] = $this->handlers;
$step_output = call_user_func( $handler, $resolved, $handler_context );
Expand Down Expand Up @@ -123,4 +125,43 @@ private static function string_value( $value ): string {

return '';
}

/**
* @param mixed $handler Handler candidate.
* @return array<string, string>|null
*/
private static function describe_handler( $handler ): ?array {
if ( is_string( $handler ) ) {
return array(
'type' => 'function',
'name' => $handler,
);
}

if ( is_array( $handler ) && isset( $handler[0], $handler[1] ) ) {
$class_or_object = is_object( $handler[0] ) ? get_class( $handler[0] ) : self::string_value( $handler[0] );
$method = self::string_value( $handler[1] );

return array(
'type' => 'method',
'name' => $class_or_object . '::' . $method,
);
}

if ( $handler instanceof \Closure ) {
return array(
'type' => 'closure',
'name' => 'Closure',
);
}

if ( is_object( $handler ) && method_exists( $handler, '__invoke' ) ) {
return array(
'type' => 'invokable',
'name' => get_class( $handler ),
);
}

return null;
}
}
33 changes: 23 additions & 10 deletions src/Workflows/register-agents-workflow-abilities.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ function agents_run_workflow_input_schema(): array {
),
'options' => array(
'type' => 'object',
'description' => 'Runtime options forwarded to the runner. Recognized keys: run_id, continue_on_error, metadata.',
'description' => 'Runtime options forwarded to the runner. Recognized keys: run_id, continue_on_error, metadata, evidence_refs.',
'default' => array(),
),
),
Expand All @@ -350,18 +350,31 @@ function agents_run_workflow_output_schema(): array {
'type' => 'object',
'required' => array( 'run_id', 'workflow_id', 'status' ),
'properties' => array(
'run_id' => array( 'type' => 'string' ),
'workflow_id' => array( 'type' => 'string' ),
'status' => array(
'run_id' => array( 'type' => 'string' ),
'workflow_id' => array( 'type' => 'string' ),
'status' => array(
'type' => 'string',
'enum' => array( 'pending', 'running', 'succeeded', 'failed', 'skipped' ),
),
'output' => array( 'type' => 'object' ),
'steps' => array( 'type' => 'array' ),
'error' => array( 'type' => array( 'object', 'null' ) ),
'started_at' => array( 'type' => 'integer' ),
'ended_at' => array( 'type' => 'integer' ),
'metadata' => array( 'type' => 'object' ),
'output' => array( 'type' => 'object' ),
'steps' => array( 'type' => 'array' ),
'error' => array( 'type' => array( 'object', 'null' ) ),
'started_at' => array( 'type' => 'integer' ),
'ended_at' => array( 'type' => 'integer' ),
'metadata' => array( 'type' => 'object' ),
'evidence_refs' => array(
'type' => 'array',
'description' => 'Neutral JSON-serializable artifact/log references owned by the host runtime.',
),
'replay' => array(
'type' => 'object',
'properties' => array(
'run_record_schema_version' => array( 'type' => 'integer' ),
'workflow_spec_version' => array( 'type' => 'string' ),
'workflow_spec_hash' => array( 'type' => 'string' ),
'workflow_spec_snapshot' => array( 'type' => 'object' ),
),
),
),
);
}
Expand Down
12 changes: 12 additions & 0 deletions tests/agents-workflow-ability-smoke.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ function smoke_assert( $expected, $actual, string $name, array &$failures, int &
agents_api_smoke_require_module();

use function AgentsAPI\AI\Workflows\agents_describe_workflow;
use function AgentsAPI\AI\Workflows\agents_run_workflow_input_schema;
use function AgentsAPI\AI\Workflows\agents_run_workflow_output_schema;
use function AgentsAPI\AI\Workflows\agents_run_workflow_dispatch;
use function AgentsAPI\AI\Workflows\agents_run_workflow_permission;
use function AgentsAPI\AI\Workflows\agents_validate_workflow;
Expand All @@ -86,6 +88,16 @@ function smoke_assert( $expected, $actual, string $name, array &$failures, int &
add_filter( 'agents_run_workflow_permission', static fn() => false );
smoke_assert( false, agents_run_workflow_permission( array() ), 'filter can override permission to deny', $failures, $passes );

// ─── run-workflow schemas ────────────────────────────────────────────

$input_schema = agents_run_workflow_input_schema();
$output_schema = agents_run_workflow_output_schema();
smoke_assert( true, str_contains( $input_schema['properties']['options']['description'] ?? '', 'evidence_refs' ), 'input schema documents evidence_refs runtime option', $failures, $passes );
smoke_assert( 'array', $output_schema['properties']['evidence_refs']['type'] ?? '', 'output schema exposes evidence_refs', $failures, $passes );
smoke_assert( 'object', $output_schema['properties']['replay']['type'] ?? '', 'output schema exposes replay metadata', $failures, $passes );
smoke_assert( 'integer', $output_schema['properties']['replay']['properties']['run_record_schema_version']['type'] ?? '', 'replay schema includes schema version', $failures, $passes );
smoke_assert( 'string', $output_schema['properties']['replay']['properties']['workflow_spec_hash']['type'] ?? '', 'replay schema includes spec hash', $failures, $passes );

// ─── validate-workflow ───────────────────────────────────────────────

$valid = agents_validate_workflow(
Expand Down
15 changes: 12 additions & 3 deletions tests/workflow-runner-smoke.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,9 +247,10 @@ static function ( array $input ): \WP_Error {

$spec = WP_Agent_Workflow_Spec::from_array(
array(
'id' => 'demo/transform',
'inputs' => array( 'text' => array( 'type' => 'string', 'required' => true ) ),
'steps' => array(
'id' => 'demo/transform',
'version' => '1.2.3',
'inputs' => array( 'text' => array( 'type' => 'string', 'required' => true ) ),
'steps' => array(
array(
'id' => 'upper',
'type' => 'ability',
Expand Down Expand Up @@ -277,6 +278,13 @@ static function ( array $input ): \WP_Error {
smoke_assert( '<<HELLO>>', $result->get_output()['last']['wrapped'], 'final output exposes last step', $failures, $passes );
smoke_assert( true, count( $recorder->writes ) >= 3, 'recorder hit at least 3 times (start + per-step updates + final)', $failures, $passes );
smoke_assert( 'start', $recorder->writes[0]['op'], 'recorder start fires first', $failures, $passes );
smoke_assert( 'demo/uppercase', $result->get_steps()[0]['resolved_step']['ability'] ?? '', 'step record includes resolved step data', $failures, $passes );
smoke_assert( 'HELLO', $result->get_steps()[1]['resolved_step']['args']['inner'] ?? '', 'resolved step data includes expanded bindings', $failures, $passes );
smoke_assert( 'method', $result->get_steps()[0]['handler']['type'] ?? '', 'step record includes handler metadata', $failures, $passes );
smoke_assert( 1, $result->get_replay_metadata()['run_record_schema_version'] ?? 0, 'replay metadata includes run record schema version', $failures, $passes );
smoke_assert( '1.2.3', $result->get_replay_metadata()['workflow_spec_version'] ?? '', 'replay metadata includes workflow spec version', $failures, $passes );
smoke_assert( true, 64 === strlen( $result->get_replay_metadata()['workflow_spec_hash'] ?? '' ), 'replay metadata includes sha256 spec hash', $failures, $passes );
smoke_assert( $spec->to_array(), $result->get_replay_metadata()['workflow_spec_snapshot'] ?? array(), 'replay metadata includes workflow spec snapshot', $failures, $passes );

// ─── Evidence refs stay first-class through results and recorders ─────

Expand Down Expand Up @@ -312,6 +320,7 @@ static function ( array $input ): \WP_Error {
smoke_assert( $evidence_refs, $evidence_result->get_evidence_refs(), 'result exposes first-class evidence refs', $failures, $passes );
smoke_assert( $evidence_result->to_array(), $roundtrip->to_array(), 'run result round-trips through to_array/from_array', $failures, $passes );
smoke_assert( $evidence_refs, $last_write['result']['evidence_refs'] ?? array(), 'recorder update preserves evidence refs', $failures, $passes );
smoke_assert( $spec->to_array(), $last_write['result']['replay']['workflow_spec_snapshot'] ?? array(), 'recorder update preserves replay metadata', $failures, $passes );
smoke_assert( true, is_string( json_encode( $evidence_result->get_evidence_refs() ) ), 'evidence refs are JSON-serializable', $failures, $passes );

// ─── Failed step short-circuits ───────────────────────────────────────
Expand Down
Loading